- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: 6 new changesets
by commits-noreply＠bitbucket.org 25 Jul '14

25 Jul '14

6 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/aed2268f79d1/ Changeset: aed2268f79d1 User: jgoecks Date: 2014-07-24 19:54:50 Summary: Trackster: variable clean up for read track painter, preventing features from being cutoff at the bottom. Affected #: 1 file diff -r 9d8967752787556b3d814e086fe2bd9eb1992170 -r aed2268f79d13f5604d69db109c20374cf311511 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -783,7 +783,9 @@ char_width_px = ctx.canvas.manager.char_width_px, block_color = (strand === "+" ? this.prefs.detail_block_color : this.prefs.reverse_strand_color), pack_mode = (mode === 'Pack'), - paint_utils = new ReadPainterUtils(ctx, (pack_mode ? PACK_FEATURE_HEIGHT : SQUISH_FEATURE_HEIGHT), w_scale, mode), + draw_height = (pack_mode ? PACK_FEATURE_HEIGHT : SQUISH_FEATURE_HEIGHT), + rect_y = y_start + 1, + paint_utils = new ReadPainterUtils(ctx, draw_height, w_scale, mode), drawing_blocks = []; // Keep list of items that need to be drawn on top of initial drawing layer. @@ -813,10 +815,7 @@ // Draw read base as rectangle. ctx.fillStyle = block_color; - ctx.fillRect(s_start, - y_start + (pack_mode ? 1 : 4 ), - s_end - s_start, - (pack_mode ? PACK_FEATURE_HEIGHT : SQUISH_FEATURE_HEIGHT)); + ctx.fillRect(s_start, rect_y, s_end - s_start, draw_height); } } @@ -893,10 +892,7 @@ } // Require a minimum w_scale so that variants are only drawn when somewhat zoomed in. else if (w_scale > 0.05) { - ctx.fillRect(c_start - gap, - y_start + (pack_mode ? 1 : 4), - Math.max( 1, Math.round(w_scale) ), - (pack_mode ? PACK_FEATURE_HEIGHT : SQUISH_FEATURE_HEIGHT)); + ctx.fillRect(c_start - gap, rect_y, Math.max( 1, Math.round(w_scale) ), draw_height); } } } @@ -913,7 +909,7 @@ base_offset += cig_len; break; case "D": // Deletion. - paint_utils.draw_deletion(s_start, y_start + (pack_mode ? 1 : 4), cig_len); + paint_utils.draw_deletion(s_start, rect_y, cig_len); base_offset += cig_len; break; case "I": // Insertion. https://bitbucket.org/galaxy/galaxy-central/commits/9f94c75733e9/ Changeset: 9f94c75733e9 User: jgoecks Date: 2014-07-24 20:34:53 Summary: Client-side visualization framework: remove extend aliasing. Affected #: 1 file diff -r aed2268f79d13f5604d69db109c20374cf311511 -r 9f94c75733e9e61f94658ecdf436d57f70eec9a6 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -1,7 +1,5 @@ define( ["libs/underscore"], function( _ ) { -var extend = _.extend; - /** * Compute the type of overlap between two regions. They are assumed to be on the same chrom/contig. * The overlap is computed relative to the second region; hence, OVERLAP_START indicates that the first @@ -128,7 +126,7 @@ this.view_start = view_start; this.view_end = view_end; // Drawing prefs - this.prefs = extend({}, this.default_prefs, prefs); + this.prefs = _.extend({}, this.default_prefs, prefs); this.mode = mode; }; @@ -354,7 +352,7 @@ FeaturePainter.prototype.default_prefs = { block_color: "#FFF", connector_color: "#FFF" }; -extend(FeaturePainter.prototype, { +_.extend(FeaturePainter.prototype, { get_required_height: function(rows_required, width) { // y_scale is the height per row var required_height = this.get_row_height(), @@ -451,7 +449,7 @@ this.draw_individual_connectors = false; }; -extend(LinkedFeaturePainter.prototype, FeaturePainter.prototype, { +_.extend(LinkedFeaturePainter.prototype, FeaturePainter.prototype, { /** * Height of a single row, depends on mode @@ -692,7 +690,7 @@ this.base_color_fn = base_color_fn; }; -extend(ReadPainter.prototype, FeaturePainter.prototype, { +_.extend(ReadPainter.prototype, FeaturePainter.prototype, { /** * Returns height based on mode. */ @@ -1077,7 +1075,7 @@ this.draw_individual_connectors = true; }; -extend(ArcLinkedFeaturePainter.prototype, FeaturePainter.prototype, LinkedFeaturePainter.prototype, { +_.extend(ArcLinkedFeaturePainter.prototype, FeaturePainter.prototype, LinkedFeaturePainter.prototype, { calculate_longest_feature_length: function () { var longest_feature_length = 0; @@ -1336,7 +1334,7 @@ this.delete_details_thickness = 0.2; }; -extend(ReadPainterUtils.prototype, { +_.extend(ReadPainterUtils.prototype, { /** * Draw deletion of base(s). * @param draw_detail if true, drawing in detail and deletion is drawn more subtly @@ -1358,7 +1356,7 @@ this.divider_height = 1; }; -extend(VariantPainter.prototype, Painter.prototype, { +_.extend(VariantPainter.prototype, Painter.prototype, { /** * Height of a single row, depends on mode */ https://bitbucket.org/galaxy/galaxy-central/commits/537e1c3b80ee/ Changeset: 537e1c3b80ee User: jgoecks Date: 2014-07-25 15:38:05 Summary: Trackster: better handling for drawing features across tiles. Affected #: 2 files diff -r 9f94c75733e9e61f94658ecdf436d57f70eec9a6 -r 537e1c3b80eec65f83871272f49f6de6393523c9 static/scripts/viz/trackster/painters.js --- a/static/scripts/viz/trackster/painters.js +++ b/static/scripts/viz/trackster/painters.js @@ -1055,10 +1055,10 @@ ctx.fillStyle = this.prefs.label_color; if (tile_low === 0 && f_start - ctx.measureText(feature_name).width < 0) { ctx.textAlign = "left"; - ctx.fillText(feature_name, f_end + LABEL_SPACING, y_start + 8, this.max_label_length); + ctx.fillText(feature_name, f_end + LABEL_SPACING, y_start + 9, this.max_label_length); } else { ctx.textAlign = "right"; - ctx.fillText(feature_name, f_start - LABEL_SPACING, y_start + 8, this.max_label_length); + ctx.fillText(feature_name, f_start - LABEL_SPACING, y_start + 9, this.max_label_length); } } diff -r 9f94c75733e9e61f94658ecdf436d57f70eec9a6 -r 537e1c3b80eec65f83871272f49f6de6393523c9 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1968,6 +1968,7 @@ * Sets up support for popups. */ FeatureTrackTile.prototype.predisplay_actions = function() { + /* // // Add support for popups. // @@ -2062,6 +2063,7 @@ .mouseleave(function() { $(this).parents(".track-content").children(".overlay").children(".feature-popup").remove(); }); +*/ }; /** @@ -3722,11 +3724,22 @@ }); }); - // Draw features on each tile. + // Draw incomplete features on each tile. var self = this; _.each(tiles, function(tile) { - self.draw_tile({ 'data': _.values(all_incomplete_features) }, tile.canvas.getContext('2d'), - tile.mode, tile.region, w_scale, tile.seq_data, true); + // To draw incomplete features, copy original canvas and then draw incomplete features + // on the canvas. + var features = { data: _.values( all_incomplete_features ) }, + canvas = self.view.canvas_manager.new_canvas(); + canvas.height = self.get_canvas_height(features, tile.mode, tile.w_scale, 100); + canvas.width = tile.canvas.width; + canvas.getContext('2d').drawImage(tile.canvas, 0, 0); + canvas.getContext('2d').translate(track.left_offset, 0); + var new_tile = self.draw_tile(features, canvas.getContext('2d'), + tile.mode, tile.region, tile.w_scale, tile.seq_data); + $(new_tile.canvas).addClass('incomplete_features'); + $(tile.canvas).replaceWith($(new_tile.canvas)); + tile.canvas = canvas; }); } https://bitbucket.org/galaxy/galaxy-central/commits/4ca83b9de85f/ Changeset: 4ca83b9de85f User: jgoecks Date: 2014-07-25 15:45:28 Summary: Trackster: disable track popups because they are slow and don't work well. Affected #: 1 file diff -r 537e1c3b80eec65f83871272f49f6de6393523c9 -r 4ca83b9de85f10b371c457fff1e06470f1a1944c static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1969,6 +1969,7 @@ */ FeatureTrackTile.prototype.predisplay_actions = function() { /* + FIXME: use a canvas library to handle popups. // // Add support for popups. // @@ -2063,7 +2064,7 @@ .mouseleave(function() { $(this).parents(".track-content").children(".overlay").children(".feature-popup").remove(); }); -*/ + */ }; /** https://bitbucket.org/galaxy/galaxy-central/commits/b2613d9978a4/ Changeset: b2613d9978a4 User: jgoecks Date: 2014-07-25 15:48:31 Summary: Trackster: better variable naming. Affected #: 1 file diff -r 4ca83b9de85f10b371c457fff1e06470f1a1944c -r b2613d9978a499db6e905ba8f4e452029b1aff61 static/scripts/viz/trackster/tracks.js --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -3731,16 +3731,16 @@ // To draw incomplete features, copy original canvas and then draw incomplete features // on the canvas. var features = { data: _.values( all_incomplete_features ) }, - canvas = self.view.canvas_manager.new_canvas(); - canvas.height = self.get_canvas_height(features, tile.mode, tile.w_scale, 100); - canvas.width = tile.canvas.width; - canvas.getContext('2d').drawImage(tile.canvas, 0, 0); - canvas.getContext('2d').translate(track.left_offset, 0); - var new_tile = self.draw_tile(features, canvas.getContext('2d'), + new_canvas = self.view.canvas_manager.new_canvas(); + new_canvas.height = self.get_canvas_height(features, tile.mode, tile.w_scale, 100); + new_canvas.width = tile.canvas.width; + new_canvas.getContext('2d').drawImage(tile.canvas, 0, 0); + new_canvas.getContext('2d').translate(track.left_offset, 0); + var new_tile = self.draw_tile(features, new_canvas.getContext('2d'), tile.mode, tile.region, tile.w_scale, tile.seq_data); $(new_tile.canvas).addClass('incomplete_features'); $(tile.canvas).replaceWith($(new_tile.canvas)); - tile.canvas = canvas; + tile.canvas = new_canvas; }); } https://bitbucket.org/galaxy/galaxy-central/commits/8b98752f4fbc/ Changeset: 8b98752f4fbc User: jgoecks Date: 2014-07-25 15:48:56 Summary: Automated merge Affected #: 13 files diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -177,6 +177,7 @@ <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/><datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" description="Any data in tab delimited format (tabular)." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Tabular_.28tab_delimited.29"/><datatype extension="twobit" type="galaxy.datatypes.binary:TwoBit" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="sqlite" type="galaxy.datatypes.binary:SQlite" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true" description="Any text file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Plain_text"/><datatype extension="linecount" type="galaxy.datatypes.data:LineCount" display_in_upload="false"/><datatype extension="memexml" type="galaxy.datatypes.xml:MEMEXml" mimetype="application/xml" display_in_upload="true"/> @@ -262,6 +263,7 @@ --><sniffer type="galaxy.datatypes.tabular:Vcf"/><sniffer type="galaxy.datatypes.binary:TwoBit"/> + <sniffer type="galaxy.datatypes.binary:SQlite"/><sniffer type="galaxy.datatypes.binary:Bam"/><sniffer type="galaxy.datatypes.binary:Sff"/><sniffer type="galaxy.datatypes.xml:Phyloxml"/> diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -12,6 +12,7 @@ import subprocess import tempfile import zipfile +import sqlite3 from urllib import urlencode, quote_plus from galaxy import eggs @@ -545,3 +546,45 @@ return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) + + +(a)dataproviders.decorators.has_dataproviders +class SQlite ( Binary ): + file_ext = "sqlite" + + # Connects and runs a query that should work on any real database + # If the file is not sqlite, an exception will be thrown and the sniffer will return false + def sniff( self, filename ): + try: + conn = sqlite3.connect(filename) + schema_version=conn.cursor().execute("pragma schema_version").fetchone() + conn.close() + if schema_version is not None: + return True + return False + except: + return False + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = "SQLite Database" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "SQLite Database (%s)" % ( data.nice_size( dataset.get_size() ) ) + + + @dataproviders.decorators.dataprovider_factory( 'sqlite', dataproviders.dataset.SQliteDataProvider.settings ) + def sqlite_dataprovider( self, dataset, **settings ): + dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) + return dataproviders.dataset.SQliteDataProvider( dataset_source, **settings ) + + +Binary.register_sniffable_binary_format("sqlite","sqlite",SQlite) + diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -11,6 +11,8 @@ import line import column import external +import sqlite3 +import re from galaxy import eggs eggs.require( 'bx-python' ) @@ -700,3 +702,39 @@ #TODO: as samtools - need more info on output format raise NotImplementedError() super( BGzipTabixDataProvider, self ).__init__( dataset, **kwargs ) + + + +class SQliteDataProvider ( base.DataProvider ): + """ + Data provider that uses a sqlite database file as its source. + + Allows any query to be run and returns the resulting rows as sqlite3 row objects + """ + settings = { + 'query' : 'str' + } + + def __init__( self, source, query=None, **kwargs ): + self.query=query + self.connection = sqlite3.connect(source.dataset.file_name); + self.connection.row_factory = sqlite3.Row + super( SQliteDataProvider, self ).__init__( source, **kwargs ) + + def query_matches_whitelist(self,query): + if re.match("select ",query,re.IGNORECASE): + if re.search("^([^\"]|\"[^\"]*\")*?;",query) or re.search("^([^\']|\'[^\']*\')*?;",query): + return False + else: + return True + return False + + + + def __iter__( self ): + if (self.query is not None) and self.query_matches_whitelist(self.query): + for row in self.connection.cursor().execute(self.query): + yield row + else: + yield + diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -34,6 +34,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.util.directory_hash import directory_hash_id +from galaxy.util.sanitize_html import sanitize_html from galaxy.web.framework.helpers import to_unicode from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField, PasswordField, SelectField, TextArea, TextField, WorkflowField, @@ -2589,8 +2590,8 @@ class DatasetCollection( object, Dictifiable, UsesAnnotations ): """ """ - dict_collection_visible_keys = ( 'id', 'name', 'collection_type' ) - dict_element_visible_keys = ( 'id', 'name', 'collection_type' ) + dict_collection_visible_keys = ( 'id', 'collection_type' ) + dict_element_visible_keys = ( 'id', 'collection_type' ) def __init__( self, @@ -3703,23 +3704,27 @@ self.country = country self.phone = phone def get_html(self): + # This should probably be deprecated eventually. It should currently + # sanitize. + # TODO Find out where else uses this and replace with + # templates html = '' if self.name: - html = html + self.name + html = html + sanitize_html(self.name) if self.institution: - html = html + '<br/>' + self.institution + html = html + '<br/>' + sanitize_html(self.institution) if self.address: - html = html + '<br/>' + self.address + html = html + '<br/>' + sanitize_html(self.address) if self.city: - html = html + '<br/>' + self.city + html = html + '<br/>' + sanitize_html(self.city) if self.state: - html = html + ' ' + self.state + html = html + ' ' + sanitize_html(self.state) if self.postal_code: - html = html + ' ' + self.postal_code + html = html + ' ' + sanitize_html(self.postal_code) if self.country: - html = html + '<br/>' + self.country + html = html + '<br/>' + sanitize_html(self.country) if self.phone: - html = html + '<br/>' + 'Phone: ' + self.phone + html = html + '<br/>' + 'phone: ' + sanitize_html(self.phone) return html class UserOpenID( object ): diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1391,7 +1391,7 @@ # Load parameters (optional) input_elem = root.find("inputs") enctypes = set() - if input_elem: + if input_elem is not None: # Handle properties of the input form self.check_values = string_as_bool( input_elem.get("check_values", self.check_values ) ) self.nginx_upload = string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) ) diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1712,10 +1712,15 @@ if self.__display_multirun_option(): # Select multiple datasets, run multiple jobs. multirun_key = "%s|__multirun__" % self.name + collection_multirun_key = "%s|__collection_multirun__" % self.name if multirun_key in (other_values or {}): multirun_value = listify( other_values[ multirun_key ] ) if multirun_value and len( multirun_value ) > 1: default_field = "select_multiple" + elif collection_multirun_key in (other_values or {}): + multirun_value = listify( other_values[ collection_multirun_key ] ) + if multirun_value: + default_field = "select_collection" else: multirun_value = value multi_dataset_matcher = DatasetMatcher( trans, self, multirun_value, other_values ) @@ -2014,9 +2019,17 @@ default_field = "select_single_collection" fields = odict() + collection_multirun_key = "%s|__collection_multirun__" % self.name + if collection_multirun_key in (other_values or {}): + multirun_value = other_values[ collection_multirun_key ] + if multirun_value: + default_field = "select_map_over_collections" + else: + multirun_value = value + history = self._get_history( trans ) fields[ "select_single_collection" ] = self._get_single_collection_field( trans=trans, history=history, value=value, other_values=other_values ) - fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=value, other_values=other_values ) + fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=multirun_value, other_values=other_values ) return self._switch_fields( fields, default_field=default_field ) diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -48,10 +48,10 @@ from .inflection import Inflector, English inflector = Inflector(English) -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) _lock = threading.RLock() -CHUNK_SIZE = 65536 #64k +CHUNK_SIZE = 65536 # 64k DATABASE_MAX_STRING_SIZE = 32768 DATABASE_MAX_STRING_SIZE_PRETTY = '32K' @@ -62,6 +62,7 @@ NULL_CHAR = '\000' BINARY_CHARS = [ NULL_CHAR ] + def is_multi_byte( chars ): for char in chars: try: @@ -69,18 +70,15 @@ except UnicodeDecodeError: # Probably binary return False - if wchartype.is_asian( char ) or \ - wchartype.is_full_width( char ) or \ - wchartype.is_kanji( char ) or \ - wchartype.is_hiragana( char ) or \ - wchartype.is_katakana( char ) or \ - wchartype.is_half_katakana( char ) or \ - wchartype.is_hangul( char ) or \ - wchartype.is_full_digit( char ) or \ - wchartype.is_full_letter( char ): + if ( wchartype.is_asian( char ) or wchartype.is_full_width( char ) or + wchartype.is_kanji( char ) or wchartype.is_hiragana( char ) or + wchartype.is_katakana( char ) or wchartype.is_half_katakana( char ) + or wchartype.is_hangul( char ) or wchartype.is_full_digit( char ) + or wchartype.is_full_letter( char )): return True return False + def is_binary( value, binary_chars=None ): """ File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.). @@ -99,6 +97,7 @@ return True return False + def get_charset_from_http_headers( headers, default=None ): rval = headers.get('content-type', None ) if rval and 'charset=' in rval: @@ -107,16 +106,18 @@ return rval return default + def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" def caller(*params, **kparams): - _lock.acquire(True) # Wait + _lock.acquire(True) # Wait try: return func(*params, **kparams) finally: _lock.release() return caller + def file_iter(fname, sep=None): """ This generator iterates over a file and yields its lines @@ -131,6 +132,7 @@ if line and line[0] != '#': yield line.split(sep) + def file_reader( fp, chunk_size=CHUNK_SIZE ): """This generator yields the open fileobject in chunks (default 64k). Closes the file at the end""" while 1: @@ -140,6 +142,7 @@ yield data fp.close() + def unique_id(KEY_SIZE=128): """ Generates an unique id @@ -148,8 +151,8 @@ >>> len(set(ids)) 1000 """ - id = str( random.getrandbits( KEY_SIZE ) ) - return md5(id).hexdigest() + return md5(str( random.getrandbits( KEY_SIZE ) )).hexdigest() + def parse_xml(fname): """Returns a parsed xml tree""" @@ -163,6 +166,7 @@ tree = ElementTree.fromstring(xml_string) return tree + def xml_to_string( elem, pretty=False ): """Returns a string from an xml tree""" if pretty: @@ -170,12 +174,13 @@ try: return ElementTree.tostring( elem ) except TypeError, e: - #assume this is a comment + # we assume this is a comment if hasattr( elem, 'text' ): return "\n" % ( elem.text ) else: raise e + def xml_element_compare( elem1, elem2 ): if not isinstance( elem1, dict ): elem1 = xml_element_to_dict( elem1 ) @@ -183,9 +188,11 @@ elem2 = xml_element_to_dict( elem2 ) return elem1 == elem2 + def xml_element_list_compare( elem_list1, elem_list2 ): return [ xml_element_to_dict( elem ) for elem in elem_list1 ] == [ xml_element_to_dict( elem ) for elem in elem_list2 ] + def xml_element_to_dict( elem ): rval = {} if elem.attrib: @@ -220,7 +227,6 @@ return rval - def pretty_print_xml( elem, level=0 ): pad = ' ' i = "\n" + level * pad @@ -238,26 +244,28 @@ elem.tail = i + pad return elem + def get_file_size( value, default=None ): try: - #try built-in + # try built-in return os.path.getsize( value ) except: try: - #try built-in one name attribute + # try built-in one name attribute return os.path.getsize( value.name ) except: try: - #try tell() of end of object + # try tell() of end of object offset = value.tell() value.seek( 0, 2 ) rval = value.tell() value.seek( offset ) return rval except: - #return default value + # return default value return default + def shrink_stream_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): rval = '' if get_file_size( value ) > size: @@ -292,6 +300,7 @@ rval += data return rval + def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): if len( value ) > size: len_join_by = len( join_by ) @@ -311,29 +320,30 @@ value = "%s%s%s" % ( value[:left_index], join_by, value[-right_index:] ) return value + def pretty_print_json(json_data, is_json_string=False): if is_json_string: json_data = json.from_json_string(json_data) return json.to_json_string(json_data, sort_keys=True, indent=4) # characters that are valid -valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") +valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") # characters that are allowed but need to be escaped -mapped_chars = { '>' :'__gt__', - '<' :'__lt__', - "'" :'__sq__', - '"' :'__dq__', - '[' :'__ob__', - ']' :'__cb__', - '{' :'__oc__', - '}' :'__cc__', - '@' : '__at__', - '\n' : '__cn__', - '\r' : '__cr__', - '\t' : '__tc__', - '#' : '__pd__' - } +mapped_chars = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '@': '__at__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + def restore_text(text): """Restores sanitized text""" @@ -343,6 +353,7 @@ text = text.replace(value, key) return text + def sanitize_text(text): """ Restricts the characters that are allowed in text; accepts both strings @@ -353,6 +364,7 @@ elif isinstance( text, list ): return [ _sanitize_text_helper(t) for t in text ] + def _sanitize_text_helper(text): """Restricts the characters that are allowed in a string""" @@ -363,9 +375,10 @@ elif c in mapped_chars: out.append(mapped_chars[c]) else: - out.append('X') # makes debugging easier + out.append('X') # makes debugging easier return ''.join(out) + def sanitize_param(value): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): @@ -373,10 +386,12 @@ elif isinstance( value, list ): return map(sanitize_text, value) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] + + def sanitize_for_filename( text, default=None ): """ Restricts the characters that are allowed in a filename portion; Returns default value or a unique id string if result is not a valid name. @@ -512,7 +527,7 @@ def __init__( self, params, sanitize=True ): if sanitize: for key, value in params.items(): - if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: #sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. + if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: # sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. self.__dict__[ key ] = sanitize_param( value ) else: self.__dict__[ key ] = value @@ -525,7 +540,7 @@ """ flat = [] for key, value in self.__dict__.items(): - if type(value) == type([]): + if isinstance(value, list): for v in value: flat.append( (key, v) ) else: @@ -551,16 +566,19 @@ def update(self, values): self.__dict__.update(values) + def rst_to_html( s ): """Convert a blob of reStructuredText to HTML""" log = logging.getLogger( "docutils" ) + class FakeStream( object ): def write( self, str ): if len( str ) > 0 and not str.isspace(): log.warn( str ) return unicodify( docutils.core.publish_string( s, - writer=docutils.writers.html4css1.Writer(), - settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + writer=docutils.writers.html4css1.Writer(), + settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + def xml_text(root, name=None): """Returns the text inside an element""" @@ -582,6 +600,8 @@ # asbool implementation pulled from PasteDeploy truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) + + def asbool(obj): if isinstance(obj, basestring): obj = obj.strip().lower() @@ -600,6 +620,7 @@ else: return False + def string_as_bool_or_none( string ): """ Returns True, None or False based on the argument: @@ -618,6 +639,7 @@ else: return False + def listify( item, do_strip=False ): """ Make a single item a single item list, or return a list if passed a @@ -635,6 +657,7 @@ else: return [ item ] + def commaify(amount): orig = amount new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', amount) @@ -643,7 +666,8 @@ else: return commaify(new) -def roundify(amount, sfs = 2): + +def roundify(amount, sfs=2): """ Take a number in string form and truncate to 'sfs' significant figures. """ @@ -652,6 +676,7 @@ else: return amount[0:sfs] + '0'*(len(amount) - sfs) + def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ): """ Returns a unicode string or None @@ -691,6 +716,7 @@ def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) + def string_to_object( s ): return pickle.loads( binascii.unhexlify( s ) ) @@ -743,19 +769,23 @@ return False return True + def get_ucsc_by_build(build): sites = [] for site in ucsc_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + + def get_gbrowse_sites_by_build(build): sites = [] for site in gbrowse_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + def read_dbnames(filename): """ Read build names from file """ class DBNames( list ): @@ -764,48 +794,54 @@ db_names = DBNames() try: ucsc_builds = {} - man_builds = [] #assume these are integers + man_builds = [] # assume these are integers name_to_db_base = {} for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") - #Special case of unspecified build is at top of list + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + # Special case of unspecified build is at top of list if fields[0] == "?": - db_names.insert(0,(fields[0],fields[1])) + db_names.insert(0, (fields[0], fields[1])) continue - try: #manual build (i.e. microbes) + try: # manual build (i.e. microbes) int(fields[0]) man_builds.append((fields[1], fields[0])) - except: #UCSC build + except: # UCSC build db_base = fields[0].rstrip('0123456789') if db_base not in ucsc_builds: ucsc_builds[db_base] = [] name_to_db_base[fields[1]] = db_base - #we want to sort within a species numerically by revision number + # we want to sort within a species numerically by revision number build_rev = re.compile(r'\d+$') - try: build_rev = int(build_rev.findall(fields[0])[0]) - except: build_rev = 0 - ucsc_builds[db_base].append((build_rev, fields[0],fields[1])) - except: continue + try: + build_rev = int(build_rev.findall(fields[0])[0]) + except: + build_rev = 0 + ucsc_builds[db_base].append((build_rev, fields[0], fields[1])) + except: + continue sort_names = name_to_db_base.keys() sort_names.sort() for name in sort_names: db_base = name_to_db_base[name] ucsc_builds[db_base].sort() ucsc_builds[db_base].reverse() - ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] + ucsc_builds[db_base] = [(build, name) for _, build, name in ucsc_builds[db_base]] db_names = DBNames( db_names + ucsc_builds[db_base] ) - if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) + if len( db_names ) > 1 and len( man_builds ) > 0: + db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) man_builds.sort() - man_builds = [(build, name) for name, build in man_builds] + man_builds = [(build, name) for name, build in man_builds] db_names = DBNames( db_names + man_builds ) except Exception, e: print "ERROR: Unable to read builds file:", e - if len(db_names)<1: + if len(db_names) < 1: db_names = DBNames( [( db_names.default_value, db_names.default_name )] ) return db_names + def read_ensembl( filename, ucsc ): """ Read Ensembl build names from file """ ucsc_builds = [] @@ -814,47 +850,55 @@ ensembl_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") - if fields[0] in ucsc_builds: continue + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + if fields[0] in ucsc_builds: + continue ensembl_builds.append( dict( dbkey=fields[0], release=fields[1], name=fields[2].replace( '_', ' ' ) ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ensembl_builds + def read_ncbi( filename ): """ Read NCBI build names from file """ ncbi_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") ncbi_builds.append( dict( dbkey=fields[0], name=fields[1] ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ncbi_builds + def read_build_sites( filename, check_builds=True ): """ read db names to ucsc mappings from file, this file should probably be merged with the one above """ build_sites = [] try: for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") site_name = fields[0] site = fields[1] if check_builds: site_builds = fields[2].split(",") - site_dict = {'name':site_name, 'url':site, 'builds':site_builds} + site_dict = {'name': site_name, 'url': site, 'builds': site_builds} else: - site_dict = {'name':site_name, 'url':site} + site_dict = {'name': site_name, 'url': site} build_sites.append( site_dict ) - except: continue + except: + continue except: - print "ERROR: Unable to read builds for site file %s" %filename + print "ERROR: Unable to read builds for site file %s" % filename return build_sites + def relativize_symlinks( path, start=None, followlinks=False): for root, dirs, files in os.walk( path, followlinks=followlinks ): rel_start = None @@ -871,23 +915,26 @@ os.remove( symlink_file_name ) os.symlink( rel_path, symlink_file_name ) + def stringify_dictionary_keys( in_dict ): - #returns a new dictionary - #changes unicode keys into strings, only works on top level (does not recurse) - #unicode keys are not valid for expansion into keyword arguments on method calls + # returns a new dictionary + # changes unicode keys into strings, only works on top level (does not recurse) + # unicode keys are not valid for expansion into keyword arguments on method calls out_dict = {} for key, value in in_dict.iteritems(): out_dict[ str( key ) ] = value return out_dict + def recursively_stringify_dictionary_keys( d ): if isinstance(d, dict): - return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) + return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()]) elif isinstance(d, list): return [recursively_stringify_dictionary_keys(x) for x in d] else: return d + def mkstemp_ln( src, prefix='mkstemp_ln_' ): """ From tempfile._mkstemp_inner, generate a hard link in the same dir with a @@ -904,9 +951,10 @@ return (os.path.abspath(file)) except OSError, e: if e.errno == errno.EEXIST: - continue # try again + continue # try again raise - raise IOError, (errno.EEXIST, "No usable temporary file name found") + raise IOError(errno.EEXIST, "No usable temporary file name found") + def umask_fix_perms( path, umask, unmasked_perms, gid=None ): """ @@ -923,7 +971,7 @@ try: os.chmod( path, perms ) except Exception, e: - log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), \ + log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), path, oct( perms ), oct( stat.S_IMODE( st.st_mode ) ), @@ -939,11 +987,12 @@ except: desired_group = gid current_group = st.st_gid - log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, \ + log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, path, current_group, e ) ) + def docstring_trim(docstring): """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" if not docstring: @@ -970,6 +1019,7 @@ # Return a single string: return '\n'.join(trimmed) + def nice_size(size): """ Returns a readably formatted string with the size @@ -989,14 +1039,15 @@ except: return '??? bytes' for ind, word in enumerate(words): - step = 1024 ** (ind + 1) + step = 1024 ** (ind + 1) if step > size: size = size / float(1024 ** ind) - if word == 'bytes': # No decimals for bytes + if word == 'bytes': # No decimals for bytes return "%d bytes" % size return "%.1f %s" % (size, word) return '??? bytes' + def size_to_bytes( size ): """ Returns a number of bytes if given a reasonably formatted string with the size @@ -1023,6 +1074,7 @@ elif multiple.startswith( 'b' ): return int( size ) + def send_mail( frm, to, subject, body, config ): """ Sends an email. @@ -1072,6 +1124,7 @@ s.sendmail( frm, to, msg.as_string() ) s.quit() + def force_symlink( source, link_name ): try: os.symlink( source, link_name ) @@ -1082,12 +1135,13 @@ else: raise e + def move_merge( source, target ): - #when using shutil and moving a directory, if the target exists, - #then the directory is placed inside of it - #if the target doesn't exist, then the target is made into the directory - #this makes it so that the target is always the target, and if it exists, - #the source contents are moved into the target + # when using shutil and moving a directory, if the target exists, + # then the directory is placed inside of it + # if the target doesn't exist, then the target is made into the directory + # this makes it so that the target is always the target, and if it exists, + # the source contents are moved into the target if os.path.isdir( source ) and os.path.exists( target ) and os.path.isdir( target ): for name in os.listdir( source ): move_merge( os.path.join( source, name ), os.path.join( target, name ) ) @@ -1103,7 +1157,7 @@ rv |= ord(x) ^ ord(y) return rv == 0 -galaxy_root_path = os.path.join(__path__[0], "..","..","..") +galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) @@ -1114,6 +1168,7 @@ gbrowse_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "gbrowse", "gbrowse_build_sites.txt" ) ) dlnames = dict(ucsc=ucsc_names, ensembl=ensembl_names, ncbi=ncbi_names) + def galaxy_directory(): return os.path.abspath(galaxy_root_path) diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -86,7 +86,7 @@ }).attr( 'title', selectionType['select_by'] - ); + ).data( "index", iIndex ); view.formRow().find( "label" ).append( button ); } }); @@ -114,11 +114,13 @@ } else { $("div#remap-row").css("display", "none"); } - this.formRow().find( "i" ).each(function(index, iElement) { + this.formRow().find( "i" ).each(function(_, iElement) { + var $iElement = $(iElement); + var index = $iElement.data("index"); if(index == enableIndex) { - $(iElement).css('color', 'black'); + $iElement.css('color', 'black'); } else { - $(iElement).css('color', 'Gray'); + $iElement.css('color', 'Gray'); } }); var $select = this.$( "select" ); diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 templates/webapps/galaxy/user/manage_info.mako --- a/templates/webapps/galaxy/user/manage_info.mako +++ b/templates/webapps/galaxy/user/manage_info.mako @@ -49,7 +49,7 @@ <div class="form-row"><div class="grid-header"> %for i, filter in enumerate( ['Active', 'Deleted', 'All'] ): - %if i > 0: + %if i > 0: <span>|</span> %endif %if show_filter == filter: @@ -62,11 +62,11 @@ </div><table class="grid"><tbody> - %for index, address in enumerate(addresses): + %for index, address in enumerate(addresses): <tr class="libraryRow libraryOrFolderRow" id="libraryRow"><td> - <div class="form-row"> - <label>${address.desc}:</label> + <div class="form-row"> + <label>${address.desc | h}:</label> ${address.get_html()} </div><div class="form-row"> @@ -82,7 +82,7 @@ </ul></div></td> - </tr> + </tr> %endfor </tbody></table> diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,4 +1,5 @@ import os +import re from StringIO import StringIO from galaxy.tools.parameters import grouping from galaxy.tools import test @@ -330,10 +331,11 @@ try: test_user = [ user for user in all_users if user["email"] == email ][0] except IndexError: + username = re.sub('[^a-z-]', '--', email.lower()) data = dict( email=email, password='testuser', - username='admin-user', + username=username, ) test_user = self._post( 'users', data, key=admin_key ).json() return test_user diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 test/functional/tools/collection_two_paired.xml --- /dev/null +++ b/test/functional/tools/collection_two_paired.xml @@ -0,0 +1,69 @@ +<tool id="collection_two_paired" name="collection_two_paired" version="0.1.0"> + <command> + #if $kind.collection_type == "paired" + cat $kind.f1.forward $kind.f1['reverse'] >> $out1; + cat $kind.f2.forward $kind.f2['reverse'] >> $out1; + #else + #for $i, $_ in enumerate($kind.f1): + cat $kind.f1[$i] $kind.f2[$i] >> $out1; + #end for + #end if + </command> + <inputs> + <conditional name="kind"> + <param type="select" name="collection_type"> + <option value="paired">Paired Datasets</option> + <option value="list">List of Datasets</option> + </param> + <when value="paired"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + <when value="list"> + <param name="f1" type="data_collection" collection_type="list" /> + <param name="f2" type="data_collection" collection_type="list" /> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="out1" /> + </outputs> + <tests> + <test> + <conditional name="kind"> + <param name="collection_type" value="paired" /> + <param name="f1"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + <test> + <conditional name="kind"> + <param name="collection_type" value="list" /> + <param name="f1"> + <collection type="list"> + <element name="l11" value="simple_line.txt" /> + <element name="l12" value="simple_line.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="list"> + <element name="l21" value="simple_line_alternative.txt" /> + <element name="l22" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -21,4 +21,5 @@ <tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> + <tool file="collection_two_paired.xml" /></toolbox> \ No newline at end of file diff -r b2613d9978a499db6e905ba8f4e452029b1aff61 -r 8b98752f4fbc11fb50f23b219f9037e7afa9a234 test/unit/tools/test_execution.py --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -16,13 +16,12 @@ eggs.require( "Paste" ) from paste import httpexceptions -# Tool with a repeat parameter, to test state update. -REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"><command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="text" name="param1" value="" /><repeat name="repeat1" label="Repeat 1"> - <param type="text" name="param2" value="" /> + %s </repeat></inputs><outputs> @@ -31,6 +30,10 @@ </tool> ''' +# Tool with a repeat parameter, to test state update. +REPEAT_TOOL_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="text" name="param2" value="" />''' +REPEAT_COLLECTION_PARAM_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="data_collection" name="param2" collection_type="paired" />''' + class ToolExecutionTestCase( TestCase, tools_support.UsesApp, tools_support.UsesTools ): @@ -287,13 +290,48 @@ } ) self.__assert_exeuted( template, template_vars ) - def __history_dataset_collection_for( self, hdas, id=1234 ): - collection = galaxy.model.DatasetCollection() + def test_subcollection_multirun_with_state_updates( self ): + self._init_tool( REPEAT_COLLECTION_PARAM_CONTENTS ) + hda1, hda2 = self.__add_dataset( 1 ), self.__add_dataset( 2 ) + collection = self.__history_dataset_collection_for( [ hda1, hda2 ], collection_type="list:paired" ) + collection_id = self.app.security.encode_id( collection.id ) + self.app.dataset_collections_service = Bunch( + match_collections=lambda collections: None + ) + template, template_vars = self.__handle_with_incoming( + repeat1_add="dummy", + ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert len( state.inputs[ "repeat1" ] ) == 1 + template, template_vars = self.__handle_with_incoming( state, **{ + "repeat1_0|param2|__collection_multirun__": "%s|paired" % collection_id, + "repeat1_add": "dummy", + } ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert state.inputs[ "repeat1" ][ 0 ][ "param2|__collection_multirun__" ] == "%s|paired" % collection_id + + def __history_dataset_collection_for( self, hdas, collection_type="list", id=1234 ): + collection = galaxy.model.DatasetCollection( + collection_type=collection_type, + ) to_element = lambda hda: galaxy.model.DatasetCollectionElement( collection=collection, element=hda, ) - collection.datasets = map(to_element, hdas) + elements = map(to_element, hdas) + if collection_type == "list:paired": + paired_collection = galaxy.model.DatasetCollection( + collection_type="paired", + ) + paired_collection.elements = elements + list_dce = galaxy.model.DatasetCollectionElement( + collection=collection, + element=paired_collection, + ) + elements = [ list_dce ] + + collection.elements = elements + history_dataset_collection_association = galaxy.model.HistoryDatasetCollectionAssociation( id=id, collection=collection, @@ -349,13 +387,13 @@ self.history.datasets.append( hda ) return hda - def __add_collection_dataset( self, id, *hdas ): + def __add_collection_dataset( self, id, collection_type="paired", *hdas ): hdca = galaxy.model.HistoryDatasetCollectionAssociation() hdca.id = id collection = galaxy.model.DatasetCollection() hdca.collection = collection collection.elements = [ galaxy.model.DatasetCollectionElement(element=self.__add_dataset( 1 )) ] - + collection.type = collection_type self.trans.sa_session.model_objects[ galaxy.model.HistoryDatasetCollectionAssociation ][ id ] = hdca self.history.dataset_collections.append( hdca ) return hdca Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Fix API test interactor to use unique usernames.
by commits-noreply＠bitbucket.org 25 Jul '14

25 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/15ca9a679522/ Changeset: 15ca9a679522 User: jmchilton Date: 2014-07-25 15:42:36 Summary: Fix API test interactor to use unique usernames. Should fixes API tests broken when validation of these changed recently. Affected #: 1 file diff -r 646c414e90515d78d99461e090815f1a01f18c72 -r 15ca9a6795225c9777b5f12f858e9a87754d8301 test/base/interactor.py --- a/test/base/interactor.py +++ b/test/base/interactor.py @@ -1,4 +1,5 @@ import os +import re from StringIO import StringIO from galaxy.tools.parameters import grouping from galaxy.tools import test @@ -330,10 +331,11 @@ try: test_user = [ user for user in all_users if user["email"] == email ][0] except IndexError: + username = re.sub('[^a-z-]', '--', email.lower()) data = dict( email=email, password='testuser', - username='admin-user', + username=username, ) test_user = self._post( 'users', data, key=admin_key ).json() return test_user Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Remove invalid attribute name from DatasetCollection's to_dict.
by commits-noreply＠bitbucket.org 25 Jul '14

25 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/646c414e9051/ Changeset: 646c414e9051 User: jmchilton Date: 2014-07-25 03:01:14 Summary: Remove invalid attribute name from DatasetCollection's to_dict. This wasn't breaking anything per se, but was always adding a null 'name' attribute to collections dictified representation. This is a remnant from an earlier version of this code that had the name attribute on the DatasetCollection instead of DatasetCollectionInstance. Affected #: 1 file diff -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 -r 646c414e90515d78d99461e090815f1a01f18c72 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2590,8 +2590,8 @@ class DatasetCollection( object, Dictifiable, UsesAnnotations ): """ """ - dict_collection_visible_keys = ( 'id', 'name', 'collection_type' ) - dict_element_visible_keys = ( 'id', 'name', 'collection_type' ) + dict_collection_visible_keys = ( 'id', 'collection_type' ) + dict_element_visible_keys = ( 'id', 'collection_type' ) def __init__( self, Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Merged in iracooke/galaxy-central (pull request #434)
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/56a7b27577de/ Changeset: 56a7b27577de User: dannon Date: 2014-07-25 00:29:57 Summary: Merged in iracooke/galaxy-central (pull request #434) Add sqlite datatype and corresponding dataprovider Affected #: 3 files diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -177,6 +177,7 @@ <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/><datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" description="Any data in tab delimited format (tabular)." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Tabular_.28tab_delimited.29"/><datatype extension="twobit" type="galaxy.datatypes.binary:TwoBit" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="sqlite" type="galaxy.datatypes.binary:SQlite" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true" description="Any text file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Plain_text"/><datatype extension="linecount" type="galaxy.datatypes.data:LineCount" display_in_upload="false"/><datatype extension="memexml" type="galaxy.datatypes.xml:MEMEXml" mimetype="application/xml" display_in_upload="true"/> @@ -262,6 +263,7 @@ --><sniffer type="galaxy.datatypes.tabular:Vcf"/><sniffer type="galaxy.datatypes.binary:TwoBit"/> + <sniffer type="galaxy.datatypes.binary:SQlite"/><sniffer type="galaxy.datatypes.binary:Bam"/><sniffer type="galaxy.datatypes.binary:Sff"/><sniffer type="galaxy.datatypes.xml:Phyloxml"/> diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -12,6 +12,7 @@ import subprocess import tempfile import zipfile +import sqlite3 from urllib import urlencode, quote_plus from galaxy import eggs @@ -545,3 +546,45 @@ return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) + + +(a)dataproviders.decorators.has_dataproviders +class SQlite ( Binary ): + file_ext = "sqlite" + + # Connects and runs a query that should work on any real database + # If the file is not sqlite, an exception will be thrown and the sniffer will return false + def sniff( self, filename ): + try: + conn = sqlite3.connect(filename) + schema_version=conn.cursor().execute("pragma schema_version").fetchone() + conn.close() + if schema_version is not None: + return True + return False + except: + return False + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = "SQLite Database" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "SQLite Database (%s)" % ( data.nice_size( dataset.get_size() ) ) + + + @dataproviders.decorators.dataprovider_factory( 'sqlite', dataproviders.dataset.SQliteDataProvider.settings ) + def sqlite_dataprovider( self, dataset, **settings ): + dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) + return dataproviders.dataset.SQliteDataProvider( dataset_source, **settings ) + + +Binary.register_sniffable_binary_format("sqlite","sqlite",SQlite) + diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -11,6 +11,8 @@ import line import column import external +import sqlite3 +import re from galaxy import eggs eggs.require( 'bx-python' ) @@ -700,3 +702,39 @@ #TODO: as samtools - need more info on output format raise NotImplementedError() super( BGzipTabixDataProvider, self ).__init__( dataset, **kwargs ) + + + +class SQliteDataProvider ( base.DataProvider ): + """ + Data provider that uses a sqlite database file as its source. + + Allows any query to be run and returns the resulting rows as sqlite3 row objects + """ + settings = { + 'query' : 'str' + } + + def __init__( self, source, query=None, **kwargs ): + self.query=query + self.connection = sqlite3.connect(source.dataset.file_name); + self.connection.row_factory = sqlite3.Row + super( SQliteDataProvider, self ).__init__( source, **kwargs ) + + def query_matches_whitelist(self,query): + if re.match("select ",query,re.IGNORECASE): + if re.search("^([^\"]|\"[^\"]*\")*?;",query) or re.search("^([^\']|\'[^\']*\')*?;",query): + return False + else: + return True + return False + + + + def __iter__( self ): + if (self.query is not None) and self.query_matches_whitelist(self.query): + for row in self.connection.cursor().execute(self.query): + yield row + else: + yield + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/47c6c6bb8eec/ Changeset: 47c6c6bb8eec User: iracooke Date: 2014-07-07 03:16:09 Summary: Add sqlite datatype and corresponding dataprovider Affected #: 3 files diff -r 6d50cd22380253129121e3e7359a6286ee9b1663 -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -177,6 +177,7 @@ <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/><datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" description="Any data in tab delimited format (tabular)." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Tabular_.28tab_delimited.29"/><datatype extension="twobit" type="galaxy.datatypes.binary:TwoBit" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="sqlite" type="galaxy.datatypes.binary:SQlite" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true" description="Any text file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Plain_text"/><datatype extension="linecount" type="galaxy.datatypes.data:LineCount" display_in_upload="false"/><datatype extension="memexml" type="galaxy.datatypes.xml:MEMEXml" mimetype="application/xml" display_in_upload="true"/> @@ -262,6 +263,7 @@ --><sniffer type="galaxy.datatypes.tabular:Vcf"/><sniffer type="galaxy.datatypes.binary:TwoBit"/> + <sniffer type="galaxy.datatypes.binary:SQlite"/><sniffer type="galaxy.datatypes.binary:Bam"/><sniffer type="galaxy.datatypes.binary:Sff"/><sniffer type="galaxy.datatypes.xml:Phyloxml"/> diff -r 6d50cd22380253129121e3e7359a6286ee9b1663 -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -12,6 +12,7 @@ import subprocess import tempfile import zipfile +import sqlite3 from urllib import urlencode, quote_plus from galaxy import eggs @@ -545,3 +546,45 @@ return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) + + +(a)dataproviders.decorators.has_dataproviders +class SQlite ( Binary ): + file_ext = "sqlite" + + # Connects and runs a query that should work on any real database + # If the file is not sqlite, an exception will be thrown and the sniffer will return false + def sniff( self, filename ): + try: + conn = sqlite3.connect(filename) + schema_version=conn.cursor().execute("pragma schema_version").fetchone() + conn.close() + if schema_version is not None: + return True + return False + except: + return False + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = "SQLite Database" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "SQLite Database (%s)" % ( data.nice_size( dataset.get_size() ) ) + + + @dataproviders.decorators.dataprovider_factory( 'sqlite', dataproviders.dataset.SQliteDataProvider.settings ) + def sqlite_dataprovider( self, dataset, **settings ): + dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) + return dataproviders.dataset.SQliteDataProvider( dataset_source, **settings ) + + +Binary.register_sniffable_binary_format("sqlite","sqlite",SQlite) + diff -r 6d50cd22380253129121e3e7359a6286ee9b1663 -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -11,6 +11,7 @@ import line import column import external +import sqlite3 from galaxy import eggs eggs.require( 'bx-python' ) @@ -700,3 +701,29 @@ #TODO: as samtools - need more info on output format raise NotImplementedError() super( BGzipTabixDataProvider, self ).__init__( dataset, **kwargs ) + + + +class SQliteDataProvider ( base.DataProvider ): + """ + Data provider that uses a sqlite database file as its source. + + Allows any query to be run and returns the resulting rows as sqlite3 row objects + """ + settings = { + 'query' : 'str' + } + + def __init__( self, source, query=None, **kwargs ): + self.query=query + self.connection = sqlite3.connect(source.dataset.file_name); + self.connection.row_factory = sqlite3.Row + super( SQliteDataProvider, self ).__init__( source, **kwargs ) + + def __iter__( self ): + if self.query is not None: + for row in self.connection.cursor().execute(self.query): + yield row + else: + yield + https://bitbucket.org/galaxy/galaxy-central/commits/034c0159c0cf/ Changeset: 034c0159c0cf User: iracooke Date: 2014-07-24 06:30:17 Summary: Merged galaxy/galaxy-central into default Affected #: 278 files diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b .hgignore --- a/.hgignore +++ b/.hgignore @@ -11,6 +11,9 @@ scripts/scramble/lib scripts/scramble/archives +# Python virtualenv +.venv + # Database stuff database/beaker_sessions database/community_files diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -12,7 +12,6 @@ from galaxy.visualization.data_providers.registry import DataProviderRegistry from galaxy.visualization.registry import VisualizationsRegistry from galaxy.tools.imp_exp import load_history_imp_exp_tools -from galaxy.tools.genome_index import load_genome_index_tools from galaxy.sample_tracking import external_service_types from galaxy.openid.providers import OpenIDProviders from galaxy.tools.data_manager.manager import DataManagers @@ -93,8 +92,6 @@ self.datatypes_registry.load_external_metadata_tool( self.toolbox ) # Load history import/export tools. load_history_imp_exp_tools( self.toolbox ) - # Load genome indexer tool. - load_genome_index_tools( self.toolbox ) # visualizations registry: associates resources with visualizations, controls how to render self.visualizations_registry = None if self.config.visualization_plugins_directory: diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -15,13 +15,12 @@ from galaxy.web.formatting import expand_pretty_datetime_format from galaxy.util import string_as_bool from galaxy.util import listify -from galaxy.util import parse_xml from galaxy.util.dbkeys import GenomeBuilds from galaxy import eggs -import pkg_resources log = logging.getLogger( __name__ ) + def resolve_path( path, root ): """If 'path' is relative make absolute by prepending 'root'""" if not( os.path.isabs( path ) ): @@ -40,9 +39,9 @@ self.config_dict = kwargs self.root = kwargs.get( 'root_dir', '.' ) # Collect the umask and primary gid from the environment - self.umask = os.umask( 077 ) # get the current umask - os.umask( self.umask ) # can't get w/o set, so set it back - self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster + self.umask = os.umask( 077 ) # get the current umask + os.umask( self.umask ) # can't get w/o set, so set it back + self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster # Database related configuration self.database = resolve_path( kwargs.get( "database_file", "database/universe.sqlite" ), self.root ) @@ -75,7 +74,7 @@ self.enable_unique_workflow_defaults = string_as_bool( kwargs.get( 'enable_unique_workflow_defaults', False ) ) self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) self.tool_data_path = resolve_path( kwargs.get( "tool_data_path", "tool-data" ), os.getcwd() ) - self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared','ucsc','chrom') ), self.root ) + self.len_file_path = resolve_path( kwargs.get( "len_file_path", os.path.join( self.tool_data_path, 'shared', 'ucsc', 'chrom') ), self.root ) self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) # The value of migrated_tools_config is the file reserved for containing only those tools that have been eliminated from the distribution # and moved to the tool shed. @@ -169,7 +168,7 @@ self.admin_users = kwargs.get( "admin_users", "" ) self.admin_users_list = [u.strip() for u in self.admin_users.split(',') if u] self.reset_password_length = int( kwargs.get('reset_password_length', '15') ) - self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-announce-join(a)bx.psu.edu") + self.mailing_join_addr = kwargs.get('mailing_join_addr', 'galaxy-announce-join(a)bx.psu.edu') self.error_email_to = kwargs.get( 'error_email_to', None ) self.activation_email = kwargs.get( 'activation_email', None ) self.user_activation_on = string_as_bool( kwargs.get( 'user_activation_on', False ) ) @@ -271,9 +270,9 @@ self.object_store_cache_path = resolve_path( kwargs.get( "object_store_cache_path", "database/object_store_cache" ), self.root ) # Handle AWS-specific config options for backward compatibility if kwargs.get( 'aws_access_key', None) is not None: - self.os_access_key= kwargs.get( 'aws_access_key', None ) - self.os_secret_key= kwargs.get( 'aws_secret_key', None ) - self.os_bucket_name= kwargs.get( 's3_bucket', None ) + self.os_access_key = kwargs.get( 'aws_access_key', None ) + self.os_secret_key = kwargs.get( 'aws_secret_key', None ) + self.os_bucket_name = kwargs.get( 's3_bucket', None ) self.os_use_reduced_redundancy = kwargs.get( 'use_reduced_redundancy', False ) else: self.os_access_key = kwargs.get( 'os_access_key', None ) @@ -376,6 +375,8 @@ self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) ) # visualization plugin framework self.visualization_plugins_directory = kwargs.get( 'visualization_plugins_directory', None ) + # Default chunk size for chunkable datatypes -- 64k + self.display_chunk_size = int( kwargs.get( 'display_chunk_size', 65536) ) @property def sentry_dsn_public( self ): @@ -452,19 +453,13 @@ except Exception, e: raise ConfigurationError( "Unable to create missing directory: %s\n%s" % ( path, e ) ) # Create the directories that it makes sense to create - for path in self.file_path, \ - self.new_file_path, \ - self.job_working_directory, \ - self.cluster_files_directory, \ - self.template_cache, \ - self.ftp_upload_dir, \ - self.library_import_dir, \ - self.user_library_import_dir, \ - self.nginx_upload_store, \ - './static/genetrack/plots', \ - self.whoosh_index_dir, \ - self.object_store_cache_path, \ - os.path.join( self.tool_data_path, 'shared', 'jars' ): + for path in (self.file_path, self.new_file_path, + self.job_working_directory, self.cluster_files_directory, + self.template_cache, self.ftp_upload_dir, + self.library_import_dir, self.user_library_import_dir, + self.nginx_upload_store, './static/genetrack/plots', + self.whoosh_index_dir, self.object_store_cache_path, + os.path.join( self.tool_data_path, 'shared', 'jars' )): self._ensure_directory( path ) # Check that required files exist tool_configs = self.tool_configs @@ -480,7 +475,7 @@ if key in self.deprecated_options: log.warning( "Config option '%s' is deprecated and will be removed in a future release. Please consult the latest version of the sample configuration file." % key ) - def is_admin_user( self,user ): + def is_admin_user( self, user ): """ Determine if the provided user is listed in `admin_users`. @@ -495,12 +490,13 @@ """ return resolve_path( path, self.root ) + def get_database_engine_options( kwargs, model_prefix='' ): """ Allow options for the SQLAlchemy database engine to be passed by using the prefix "database_engine_option". """ - conversions = { + conversions = { 'convert_unicode': string_as_bool, 'pool_timeout': int, 'echo': string_as_bool, @@ -522,6 +518,7 @@ rval[ key ] = value return rval + def configure_logging( config ): """ Allow some basic logging configuration to be read from ini file. @@ -556,7 +553,7 @@ root.addHandler( handler ) # If sentry is configured, also log to it if config.sentry_dsn: - pkg_resources.require( "raven" ) + eggs.require( "raven" ) from raven.handlers.logging import SentryHandler sentry_handler = SentryHandler( config.sentry_dsn ) sentry_handler.setLevel( logging.WARN ) diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/datatypes/metadata.py --- a/lib/galaxy/datatypes/metadata.py +++ b/lib/galaxy/datatypes/metadata.py @@ -126,10 +126,21 @@ rval[key] = self.spec[key].param.make_copy( value, target_context=self, source_context=to_copy ) return rval - def from_JSON_dict( self, filename, path_rewriter=None ): + def from_JSON_dict( self, filename=None, path_rewriter=None, json_dict=None ): dataset = self.parent - log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) - JSONified_dict = json.load( open( filename ) ) + if filename is not None: + log.debug( 'loading metadata from file for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) + JSONified_dict = json.load( open( filename ) ) + elif json_dict is not None: + log.debug( 'loading metadata from dict for: %s %s' % ( dataset.__class__.__name__, dataset.id ) ) + if isinstance( json_dict, basestring ): + JSONified_dict = json.loads( json_dict ) + elif isinstance( json_dict, dict ): + JSONified_dict = json_dict + else: + raise ValueError( "json_dict must be either a dictionary or a string, got %s." % ( type( json_dict ) ) ) + else: + raise ValueError( "You must provide either a filename or a json_dict" ) for name, spec in self.spec.items(): if name in JSONified_dict: from_ext_kwds = {} @@ -143,13 +154,15 @@ #metadata associated with our dataset, we'll delete it from our dataset's metadata dict del dataset._metadata[ name ] - def to_JSON_dict( self, filename ): + def to_JSON_dict( self, filename=None ): #galaxy.model.customtypes.json_encoder.encode() meta_dict = {} dataset_meta_dict = self.parent._metadata for name, spec in self.spec.items(): if name in dataset_meta_dict: meta_dict[ name ] = spec.param.to_external_value( dataset_meta_dict[ name ] ) + if filename is None: + return json.dumps( meta_dict ) json.dump( meta_dict, open( filename, 'wb+' ) ) def __getstate__( self ): diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/datatypes/tabular.py --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -25,7 +25,6 @@ # All tabular data is chunkable. CHUNKABLE = True - CHUNK_SIZE = 10000 """Add metadata elements""" MetadataElement( name="comment_lines", default=0, desc="Number of comment lines", readonly=False, optional=True, no_value=0 ) @@ -262,13 +261,13 @@ def get_chunk(self, trans, dataset, chunk): ck_index = int(chunk) f = open(dataset.file_name) - f.seek(ck_index * self.CHUNK_SIZE) + f.seek(ck_index * trans.app.config.display_chunk_size) # If we aren't at the start of the file, seek to next newline. Do this better eventually. if f.tell() != 0: cursor = f.read(1) while cursor and cursor != '\n': cursor = f.read(1) - ck_data = f.read(self.CHUNK_SIZE) + ck_data = f.read(trans.app.config.display_chunk_size) cursor = f.read(1) while cursor and ck_data[-1] != '\n': ck_data += cursor diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/exceptions/__init__.py --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -86,6 +86,11 @@ err_code = error_codes.USER_REQUEST_INVALID_PARAMETER +class AuthenticationFailed( MessageException ): + status_code = 401 + err_code = error_codes.USER_AUTHENTICATION_FAILED + + class AuthenticationRequired( MessageException ): status_code = 403 #TODO: as 401 and send WWW-Authenticate: ??? diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/exceptions/error_codes.json --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -60,6 +60,11 @@ "message": "Supplied incorrect or incompatible tool meta parameters." }, { + "name": "USER_AUTHENTICATION_FAILED", + "code": 401001, + "message": "Authentication failed, invalid credentials supplied." + }, + { "name": "USER_NO_API_KEY", "code": 403001, "message": "API authentication required for this request" diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1192,9 +1192,6 @@ param_dict = self.tool.params_from_strings( param_dict, self.app ) # Check for and move associated_files self.tool.collect_associated_files(out_data, self.working_directory) - gitd = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() - if gitd: - self.tool.collect_associated_files({'': gitd}, self.working_directory) # Create generated output children and primary datasets and add to param_dict collected_datasets = { 'children': self.tool.collect_child_datasets(out_data, self.working_directory), @@ -1248,7 +1245,6 @@ self.external_output_metadata.cleanup_external_metadata( self.sa_session ) galaxy.tools.imp_exp.JobExportHistoryArchiveWrapper( self.job_id ).cleanup_after_job( self.sa_session ) galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper( self.app, self.job_id ).cleanup_after_job() - galaxy.tools.genome_index.GenomeIndexToolWrapper( self.job_id ).postprocessing( self.sa_session, self.app ) if delete_files: self.app.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, extra_dir=str(self.job_id)) except: @@ -1351,10 +1347,8 @@ dataset_path_rewriter = self.dataset_path_rewriter job = self.get_job() - # Job output datasets are combination of history, library, jeha and gitd datasets. + # Job output datasets are combination of history, library, and jeha datasets. special = self.sa_session.query( model.JobExportHistoryArchive ).filter_by( job=job ).first() - if not special: - special = self.sa_session.query( model.GenomeIndexToolData ).filter_by( job=job ).first() false_path = None results = [] diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/actions/post.py --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -215,8 +215,17 @@ p_str += "<label for='pja__"+pja.output_name+"__RenameDatasetAction__newname'>New output name:</label>\ <input type='text' name='pja__"+pja.output_name+"__RenameDatasetAction__newname' value=''/>"; } + inputlist = []; + $.each(node.input_terminals, function(i, v){ + inputlist.push(v.name); + }); + if (inputlist !== []){ + p_str += "Available inputs are: <strong>" + inputlist.join(', ') + "</strong>"; + }else{ + p_str += "No inputs are available for templating into this action."; + } """ - return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset.") + return get_form_template(cls.name, cls.verbose_name, form, "This action will rename the result dataset. See <a href='https://wiki.galaxyproject.org/Learn/AdvancedWorkflow/Variables'>the wiki</a> for usage information.") @classmethod def get_short_str(cls, pja): diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/deferred/genome_index.py --- a/lib/galaxy/jobs/deferred/genome_index.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Module for managing genome transfer jobs. -""" -from __future__ import with_statement - -import logging, shutil, gzip, bz2, zipfile, tempfile, tarfile, sys, os - -from galaxy import eggs -from sqlalchemy import and_ -from data_transfer import * - -log = logging.getLogger( __name__ ) - -__all__ = [ 'GenomeIndexPlugin' ] - -class GenomeIndexPlugin( DataTransfer ): - - def __init__( self, app ): - super( GenomeIndexPlugin, self ).__init__( app ) - self.app = app - self.tool = app.toolbox.tools_by_id['__GENOME_INDEX__'] - self.sa_session = app.model.context.current - - def create_job( self, trans, path, indexes, dbkey, intname ): - params = dict( user=trans.user.id, path=path, indexes=indexes, dbkey=dbkey, intname=intname ) - deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'GenomeIndexPlugin', params = params ) - self.sa_session.add( deferred ) - self.sa_session.flush() - log.debug( 'Job created, id %d' % deferred.id ) - return deferred.id - - def check_job( self, job ): - log.debug( 'Job check' ) - return 'ready' - - def run_job( self, job ): - incoming = dict( path=os.path.abspath( job.params[ 'path' ] ), indexer=job.params[ 'indexes' ][0], user=job.params[ 'user' ] ) - indexjob = self.tool.execute( self, set_output_hid=False, history=None, incoming=incoming, transfer=None, deferred=job ) - job.params[ 'indexjob' ] = indexjob[0].id - job.state = self.app.model.DeferredJob.states.RUNNING - self.sa_session.add( job ) - self.sa_session.flush() - return self.app.model.DeferredJob.states.RUNNING diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/deferred/genome_transfer.py --- a/lib/galaxy/jobs/deferred/genome_transfer.py +++ /dev/null @@ -1,250 +0,0 @@ -""" -Module for managing genome transfer jobs. -""" -from __future__ import with_statement - -import logging, shutil, gzip, bz2, zipfile, tempfile, tarfile, sys - -from galaxy import eggs -from sqlalchemy import and_ - -from galaxy.util.odict import odict -from galaxy.workflow.modules import module_factory -from galaxy.jobs.actions.post import ActionBox - -from galaxy.tools.parameters import visit_input_values -from galaxy.tools.parameters.basic import DataToolParameter -from galaxy.tools.data import ToolDataTableManager - -from galaxy.datatypes.checkers import * -from galaxy.datatypes.sequence import Fasta -from data_transfer import * - -log = logging.getLogger( __name__ ) - -__all__ = [ 'GenomeTransferPlugin' ] - -class GenomeTransferPlugin( DataTransfer ): - - locations = {} - - def __init__( self, app ): - super( GenomeTransferPlugin, self ).__init__( app ) - self.app = app - self.tool = app.toolbox.tools_by_id['__GENOME_INDEX__'] - self.sa_session = app.model.context.current - tdtman = ToolDataTableManager( app.config.tool_data_path ) - xmltree = tdtman.load_from_config_file( app.config.tool_data_table_config_path, app.config.tool_data_path ) - for node in xmltree: - table = node.get('name') - location = node.findall('file')[0].get('path') - self.locations[table] = location - - def create_job( self, trans, url, dbkey, intname, indexes ): - job = trans.app.transfer_manager.new( protocol='http', url=url ) - params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', type='init_transfer', url=url, dbkey=dbkey, indexes=indexes, intname=intname, liftover=None ) - deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'GenomeTransferPlugin', params = params ) - self.sa_session.add( deferred ) - self.sa_session.flush() - return deferred.id - - def check_job( self, job ): - if job.params['type'] == 'init_transfer': - if not hasattr(job, 'transfer_job'): - job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) - else: - self.sa_session.refresh( job.transfer_job ) - if job.transfer_job.state == 'done': - transfer = job.transfer_job - transfer.state = 'downloaded' - job.params['type'] = 'extract_transfer' - self.sa_session.add( job ) - self.sa_session.add( transfer ) - self.sa_session.flush() - return self.job_states.READY - elif job.transfer_job.state == 'running': - return self.job_states.WAIT - elif job.transfer_job.state == 'new': - assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ] - self.app.transfer_manager.run( job.transfer_job ) - self.sa_session.add( job.transfer_job ) - self.sa_session.flush() - return self.job_states.WAIT - else: - log.error( "An error occurred while downloading from %s" % job.params[ 'url' ] ) - return self.job_states.INVALID - elif job.params[ 'type' ] == 'extract_transfer': - return self.job_states.READY - - def get_job_status( self, jobid ): - job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) - if 'transfer_job_id' in job.params: - if not hasattr( job, 'transfer_job' ): - job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) - else: - self.sa_session.refresh( job.transfer_job ) - return job - - def run_job( self, job ): - params = job.params - dbkey = params[ 'dbkey' ] - if not hasattr( job, 'transfer_job' ): - job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) - else: - self.sa_session.refresh( job.transfer_job ) - transfer = job.transfer_job - if params[ 'type' ] == 'extract_transfer': - CHUNK_SIZE = 2**20 - destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), job.params[ 'dbkey' ], 'seq' ) - destfile = '%s.fa' % job.params[ 'dbkey' ] - destfilepath = os.path.join( destpath, destfile ) - tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] ) - tmppath = os.path.dirname( os.path.abspath( transfer.path ) ) - if not os.path.exists( destpath ): - os.makedirs( destpath ) - protocol = job.params[ 'protocol' ] - data_type = self._check_compress( transfer.path ) - if data_type is None: - sniffer = Fasta() - if sniffer.sniff( transfer.path ): - data_type = 'fasta' - fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False ) - if data_type in [ 'tar.gzip', 'tar.bzip' ]: - fp = open( transfer.path, 'r' ) - tar = tarfile.open( mode = 'r:*', bufsize = CHUNK_SIZE, fileobj = fp ) - files = tar.getmembers() - for filename in files: - z = tar.extractfile(filename) - while 1: - try: - chunk = z.read( CHUNK_SIZE ) - except IOError: - os.close( fd ) - log.error( 'Problem decompressing compressed data' ) - exit() - if not chunk: - break - os.write( fd, chunk ) - os.write( fd, '\n' ) - os.close( fd ) - tar.close() - fp.close() - elif data_type == 'gzip': - compressed = gzip.open( transfer.path, mode = 'rb' ) - while 1: - try: - chunk = compressed.read( CHUNK_SIZE ) - except IOError: - compressed.close() - log.error( 'Problem decompressing compressed data' ) - exit() - if not chunk: - break - os.write( fd, chunk ) - os.close( fd ) - compressed.close() - elif data_type == 'bzip': - compressed = bz2.BZ2File( transfer.path, mode = 'r' ) - while 1: - try: - chunk = compressed.read( CHUNK_SIZE ) - except IOError: - compressed.close() - log.error( 'Problem decompressing compressed data' ) - exit() - if not chunk: - break - os.write( fd, chunk ) - os.close( fd ) - compressed.close() - elif data_type == 'zip': - uncompressed_name = None - unzipped = False - z = zipfile.ZipFile( transfer.path ) - z.debug = 3 - for name in z.namelist(): - if name.endswith('/'): - continue - zipped_file = z.open( name ) - while 1: - try: - chunk = zipped_file.read( CHUNK_SIZE ) - except IOError: - os.close( fd ) - log.error( 'Problem decompressing zipped data' ) - return self.app.model.DeferredJob.states.INVALID - if not chunk: - break - os.write( fd, chunk ) - zipped_file.close() - os.close( fd ) - z.close() - elif data_type == 'fasta': - uncompressed = transfer.path - else: - job.state = self.app.model.DeferredJob.states.INVALID - log.error( "Unrecognized compression format for file %s." % transfer.path ) - self.sa_session.add( job ) - self.sa_session.flush() - return - shutil.move( uncompressed, destfilepath ) - if os.path.exists( transfer.path ): - os.remove( transfer.path ) - os.chmod( destfilepath, 0644 ) - fastaline = '\t'.join( [ dbkey, dbkey, params[ 'intname' ], os.path.abspath( destfilepath ) ] ) - self._add_line( 'all_fasta', fastaline ) - if params[ 'indexes' ] is not None: - job.state = self.app.model.DeferredJob.states.WAITING - job.params[ 'indexjobs' ] = [] - else: - job.state = self.app.model.DeferredJob.states.OK - job.params[ 'type' ] = 'finish_transfer' - transfer.path = os.path.abspath(destfilepath) - transfer.state = 'done' - self.sa_session.add( job ) - self.sa_session.add( transfer ) - if transfer.state == 'done': - if params[ 'indexes' ] is not None: - for indexer in params[ 'indexes' ]: - incoming = dict(indexer=indexer, dbkey=params[ 'dbkey' ], intname=params[ 'intname' ], path=transfer.path, user=params['user'] ) - deferred = self.tool.execute( self, set_output_hid=False, history=None, incoming=incoming, transfer=transfer, deferred=job ) - job.params[ 'indexjobs' ].append( deferred[0].id ) - else: - job.state = self.app.model.DeferredJob.states.OK - self.sa_session.add( job ) - self.sa_session.flush() - return self.app.model.DeferredJob.states.OK - - def _check_compress( self, filepath ): - retval = '' - if tarfile.is_tarfile( filepath ): - retval = 'tar.' - if check_zip( filepath ): - return 'zip' - is_bzipped, is_valid = check_bz2( filepath ) - if is_bzipped and is_valid: - return retval + 'bzip' - is_gzipped, is_valid = check_gzip( filepath ) - if is_gzipped and is_valid: - return retval + 'gzip' - return None - - def _add_line( self, locfile, newline ): - filepath = self.locations[ locfile ] - origlines = [] - output = [] - comments = [] - with open( filepath, 'r' ) as destfile: - for line in destfile: - if line.startswith( '#' ): - comments.append( line.strip() ) - else: - origlines.append( line.strip() ) - if newline not in origlines: - origlines.append( newline ) - output.extend( comments ) - origlines.sort() - output.extend( origlines ) - with open( filepath, 'w+' ) as destfile: - destfile.write( '\n'.join( output ) ) - diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/deferred/liftover_transfer.py --- a/lib/galaxy/jobs/deferred/liftover_transfer.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Module for managing genome transfer jobs. -""" -from __future__ import with_statement - -import logging, shutil, gzip, tempfile, sys - -from galaxy import eggs -from sqlalchemy import and_ - -from galaxy.util.odict import odict -from galaxy.workflow.modules import module_factory -from galaxy.jobs.actions.post import ActionBox - -from galaxy.tools.parameters import visit_input_values -from galaxy.tools.parameters.basic import DataToolParameter - -from galaxy.datatypes.checkers import * - -from data_transfer import * - -log = logging.getLogger( __name__ ) - -__all__ = [ 'LiftOverTransferPlugin' ] - -class LiftOverTransferPlugin( DataTransfer ): - - locations = {} - - def __init__( self, app ): - super( LiftOverTransferPlugin, self ).__init__( app ) - self.app = app - self.sa_session = app.model.context.current - - def create_job( self, trans, url, dbkey, from_genome, to_genome, destfile, parentjob ): - job = trans.app.transfer_manager.new( protocol='http', url=url ) - params = dict( user=trans.user.id, transfer_job_id=job.id, protocol='http', - type='init_transfer', dbkey=dbkey, from_genome=from_genome, - to_genome=to_genome, destfile=destfile, parentjob=parentjob ) - deferred = trans.app.model.DeferredJob( state = self.app.model.DeferredJob.states.NEW, plugin = 'LiftOverTransferPlugin', params = params ) - self.sa_session.add( deferred ) - self.sa_session.flush() - return deferred.id - - def check_job( self, job ): - if job.params['type'] == 'init_transfer': - if not hasattr(job, 'transfer_job'): - job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) - else: - self.sa_session.refresh( job.transfer_job ) - if job.transfer_job.state == 'done': - transfer = job.transfer_job - transfer.state = 'downloaded' - job.params['type'] = 'extract_transfer' - self.sa_session.add( job ) - self.sa_session.add( transfer ) - self.sa_session.flush() - return self.job_states.READY - elif job.transfer_job.state == 'running': - return self.job_states.WAIT - elif job.transfer_job.state == 'new': - assert job.params[ 'protocol' ] in [ 'http', 'ftp', 'https' ], 'Unknown protocol %s' % job.params[ 'protocol' ] - ready = True - parent = self.sa_session.query( self.app.model.DeferredJob ).get( int( job.params[ 'parentjob' ] ) ) - if not hasattr( parent, 'transfer_job' ): - parent.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( parent.params[ 'transfer_job_id' ] ) ) - if parent.transfer_job.state not in [ 'ok', 'error', 'done' ]: - ready = False - for lo_job in parent.params[ 'liftover' ]: - liftoverjob = self.sa_session.query( self.app.model.TransferJob ).get( int( lo_job ) ) - if liftoverjob: - if liftoverjob.state not in [ 'ok', 'error', 'new', 'done' ]: - ready = False - if ready: - self.app.transfer_manager.run( job.transfer_job ) - self.sa_session.add( job.transfer_job ) - self.sa_session.flush() - return self.job_states.WAIT - else: - log.error( "An error occurred while downloading from %s" % job.transfer_job.params[ 'url' ] ) - return self.job_states.INVALID - elif job.params[ 'type' ] == 'extract_transfer': - return self.job_states.READY - - def get_job_status( self, jobid ): - job = self.sa_session.query( self.app.model.DeferredJob ).get( int( jobid ) ) - return job - - def run_job( self, job ): - params = job.params - dbkey = params[ 'dbkey' ] - source = params[ 'from_genome' ] - target = params[ 'to_genome' ] - if not hasattr( job, 'transfer_job' ): - job.transfer_job = self.sa_session.query( self.app.model.TransferJob ).get( int( job.params[ 'transfer_job_id' ] ) ) - else: - self.sa_session.refresh( job.transfer_job ) - transfer = job.transfer_job - if params[ 'type' ] == 'extract_transfer': - CHUNK_SIZE = 2**20 - destpath = os.path.join( self.app.config.get( 'genome_data_path', 'tool-data/genome' ), source, 'liftOver' ) - if not os.path.exists( destpath ): - os.makedirs( destpath ) - destfile = job.params[ 'destfile' ] - destfilepath = os.path.join( destpath, destfile ) - tmpprefix = '%s_%s_download_unzip_' % ( job.params['dbkey'], job.params[ 'transfer_job_id' ] ) - tmppath = os.path.dirname( os.path.abspath( transfer.path ) ) - if not os.path.exists( destpath ): - os.makedirs( destpath ) - fd, uncompressed = tempfile.mkstemp( prefix=tmpprefix, dir=tmppath, text=False ) - chain = gzip.open( transfer.path, 'rb' ) - while 1: - try: - chunk = chain.read( CHUNK_SIZE ) - except IOError: - os.close( fd ) - log.error( 'Problem decompressing compressed data' ) - exit() - if not chunk: - break - os.write( fd, chunk ) - os.close( fd ) - chain.close() - # Replace the gzipped file with the decompressed file if it's safe to do so - shutil.move( uncompressed, destfilepath ) - os.remove( transfer.path ) - os.chmod( destfilepath, 0644 ) - locline = '\t'.join( [ source, target, os.path.abspath( destfilepath ) ] ) - self._add_line( locline ) - job.state = self.app.model.DeferredJob.states.OK - job.params[ 'type' ] = 'finish_transfer' - transfer.path = os.path.abspath(destfilepath) - transfer.state = 'done' - parentjob = self.sa_session.query( self.app.model.DeferredJob ).get( int( job.params[ 'parentjob' ] ) ) - finished = True - for i in parentjob.params[ 'liftover' ]: - sibling = self.sa_session.query( self.app.model.DeferredJob ).get( int( i ) ) - if sibling.state not in [ 'done', 'ok', 'error' ]: - finished = False - if finished: - parentjob.state = self.app.model.DeferredJob.states.OK - self.sa_session.add( parentjob ) - self.sa_session.add( job ) - self.sa_session.add( transfer ) - self.sa_session.flush() - return self.app.model.DeferredJob.states.OK - - def _add_line( self, newline ): - filepath = 'tool-data/liftOver.loc' - origlines = [] - with open( filepath, 'r' ) as destfile: - for line in destfile: - origlines.append( line.strip() ) - if newline not in origlines: - origlines.append( newline ) - with open( filepath, 'w+' ) as destfile: - destfile.write( '\n'.join( origlines ) ) - diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/mapper.py --- a/lib/galaxy/jobs/mapper.py +++ b/lib/galaxy/jobs/mapper.py @@ -83,7 +83,7 @@ if "job" in function_arg_names or "user" in function_arg_names or "user_email" in function_arg_names or "resource_params" in function_arg_names: job = self.job_wrapper.get_job() history = job.history - user = history and history.user + user = job.user user_email = user and str(user.email) if "job" in function_arg_names: diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/jobs/metrics/formatting.py --- a/lib/galaxy/jobs/metrics/formatting.py +++ b/lib/galaxy/jobs/metrics/formatting.py @@ -15,4 +15,4 @@ elif value < 3600: return "%s minutes" % ( value / 60 ) else: - return "%s days and %s minutes" % ( value / 3600, ( value % 3600 ) / 60 ) + return "%s hours and %s minutes" % ( value / 3600, ( value % 3600 ) / 60 ) diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/managers/__init__.py --- a/lib/galaxy/managers/__init__.py +++ b/lib/galaxy/managers/__init__.py @@ -1,4 +1,33 @@ -""" 'Business logic' independent of web transactions/user context (trans) -should be pushed into models - but logic that requires the context trans -should be placed under this module. """ +Classes that manage resources (models, tools, etc.) by using the current +Transaction. + +Encapsulates the intersection of trans (or trans.sa_session), models, +and Controllers. + +Responsibilities: + model operations that involve the trans/sa_session (CRUD) + security: + ownership, accessibility + common aspect-oriented operations via new mixins: + sharable, annotatable, tagable, ratable + +Not responsible for: + encoding/decoding ids + any http gobblygook + formatting of returned data (always python structures) + formatting of raised errors + +The goal is to have Controllers only handle: + query-string/payload parsing and encoding/decoding ids + http + return formatting + +and: + control, improve namespacing in Controllers + DRY for Controller ops (define here - use in both UI/API Controllers) + +In other words, 'Business logic' independent of web transactions/user context +(trans) should be pushed into models - but logic that requires the context +trans should be placed under this module. +""" diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/managers/base.py --- /dev/null +++ b/lib/galaxy/managers/base.py @@ -0,0 +1,7 @@ + + +class ModelManager( object ): + pass + +class ModelSerializer( object ): + pass diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/managers/hdas.py --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -1,16 +1,35 @@ +""" +Manager and Serializer for HDAs. + +HistoryDatasetAssociations (HDAs) are datasets contained or created in a +history. +""" + from galaxy import exceptions -from ..managers import histories +from galaxy.managers import base as manager_base +from galaxy.managers import histories as history_manager -class HDAManager( object ): +import galaxy.web +import galaxy.datatypes.metadata +from galaxy import objectstore + +class HDAManager( manager_base.ModelManager ): + """ + Interface/service object for interacting with HDAs. + """ def __init__( self ): - self.histories_mgr = histories.HistoryManager() + """ + Set up and initialize other managers needed by hdas. + """ + self.histories_mgr = history_manager.HistoryManager() def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True ): """ + Get an HDA by its unencoded db id, checking ownership (via its history) + or accessibility (via dataset shares/permissions). """ - # this is a replacement for UsesHistoryDatasetAssociationMixin because mixins are a bad soln/structure hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( unencoded_id ) if hda is None: raise exceptions.ObjectNotFound() @@ -19,7 +38,8 @@ def secure( self, trans, hda, check_ownership=True, check_accessible=True ): """ - checks if (a) user owns item or (b) item is accessible to user. + check ownership (via its history) or accessibility (via dataset + shares/permissions). """ # all items are accessible to an admin if trans.user and trans.user_is_admin(): @@ -31,12 +51,18 @@ return hda def can_access_dataset( self, trans, hda ): + """ + Use security agent to see if current user has access to dataset. + """ current_user_roles = trans.get_current_user_roles() return trans.app.security_agent.can_access_dataset( current_user_roles, hda.dataset ) #TODO: is_owner, is_accessible def check_ownership( self, trans, hda ): + """ + Use history to see if current user owns HDA. + """ if not trans.user: #if hda.history == trans.history: # return hda @@ -51,6 +77,9 @@ "HistoryDatasetAssociation is not owned by the current user", type='error' ) def check_accessible( self, trans, hda ): + """ + Raise error if HDA is not accessible. + """ if trans.user and trans.user_is_admin(): return hda # check for access of the containing history... @@ -62,6 +91,151 @@ "HistoryDatasetAssociation is not accessible to the current user", type='error' ) def err_if_uploading( self, trans, hda ): + """ + Raise error if HDA is still uploading. + """ if hda.state == trans.model.Dataset.states.UPLOAD: raise exceptions.Conflict( "Please wait until this dataset finishes uploading" ) return hda + + def get_hda_dict( self, trans, hda ): + """ + Return full details of this HDA in dictionary form. + """ + #precondition: the user's access to this hda has already been checked + #TODO:?? postcondition: all ids are encoded (is this really what we want at this level?) + expose_dataset_path = trans.user_is_admin() or trans.app.config.expose_dataset_path + hda_dict = hda.to_dict( view='element', expose_dataset_path=expose_dataset_path ) + hda_dict[ 'api_type' ] = "file" + + # Add additional attributes that depend on trans must be added here rather than at the model level. + can_access_hda = trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), hda.dataset ) + can_access_hda = ( trans.user_is_admin() or can_access_hda ) + if not can_access_hda: + return self.get_inaccessible_hda_dict( trans, hda ) + hda_dict[ 'accessible' ] = True + + #TODO: I'm unclear as to which access pattern is right + hda_dict[ 'annotation' ] = hda.get_item_annotation_str( trans.sa_session, hda.history.user, hda ) + #annotation = getattr( hda, 'annotation', hda.get_item_annotation_str( trans.sa_session, trans.user, hda ) ) + + # ---- return here if deleted AND purged OR can't access + purged = ( hda.purged or hda.dataset.purged ) + if ( hda.deleted and purged ): + #TODO: to_dict should really go AFTER this - only summary data + return trans.security.encode_dict_ids( hda_dict ) + + if expose_dataset_path: + try: + hda_dict[ 'file_name' ] = hda.file_name + except objectstore.ObjectNotFound: + log.exception( 'objectstore.ObjectNotFound, HDA %s.', hda.id ) + + hda_dict[ 'download_url' ] = galaxy.web.url_for( 'history_contents_display', + history_id = trans.security.encode_id( hda.history.id ), + history_content_id = trans.security.encode_id( hda.id ) ) + + # indeces, assoc. metadata files, etc. + meta_files = [] + for meta_type in hda.metadata.spec.keys(): + if isinstance( hda.metadata.spec[ meta_type ].param, galaxy.datatypes.metadata.FileParameter ): + meta_files.append( dict( file_type=meta_type ) ) + if meta_files: + hda_dict[ 'meta_files' ] = meta_files + + # currently, the viz reg is optional - handle on/off + if trans.app.visualizations_registry: + hda_dict[ 'visualizations' ] = trans.app.visualizations_registry.get_visualizations( trans, hda ) + else: + hda_dict[ 'visualizations' ] = hda.get_visualizations() + #TODO: it may also be wiser to remove from here and add as API call that loads the visualizations + # when the visualizations button is clicked (instead of preloading/pre-checking) + + # ---- return here if deleted + if hda.deleted and not purged: + return trans.security.encode_dict_ids( hda_dict ) + + return trans.security.encode_dict_ids( hda_dict ) + + def get_inaccessible_hda_dict( self, trans, hda ): + """ + Return truncated serialization of HDA when inaccessible to user. + """ + return trans.security.encode_dict_ids({ + 'id' : hda.id, + 'history_id': hda.history.id, + 'hid' : hda.hid, + 'name' : hda.name, + 'state' : hda.state, + 'deleted' : hda.deleted, + 'visible' : hda.visible, + 'accessible': False + }) + + def get_hda_dict_with_error( self, trans, hda=None, history_id=None, id=None, error_msg='Error' ): + """ + Return truncated serialization of HDA when error raised getting + details. + """ + return trans.security.encode_dict_ids({ + 'id' : hda.id if hda else id, + 'history_id': hda.history.id if hda else history_id, + 'hid' : hda.hid if hda else '(unknown)', + 'name' : hda.name if hda else '(unknown)', + 'error' : error_msg, + 'state' : trans.model.Dataset.states.NEW + }) + + def get_display_apps( self, trans, hda ): + """ + Return dictionary containing new-style display app urls. + """ + display_apps = [] + for display_app in hda.get_display_applications( trans ).itervalues(): + + app_links = [] + for link_app in display_app.links.itervalues(): + app_links.append({ + 'target': link_app.url.get( 'target_frame', '_blank' ), + 'href' : link_app.get_display_url( hda, trans ), + 'text' : gettext( link_app.name ) + }) + if app_links: + display_apps.append( dict( label=display_app.name, links=app_links ) ) + + return display_apps + + def get_old_display_applications( self, trans, hda ): + """ + Return dictionary containing old-style display app urls. + """ + display_apps = [] + if not trans.app.config.enable_old_display_applications: + return display_apps + + for display_app in hda.datatype.get_display_types(): + target_frame, display_links = hda.datatype.get_display_links( hda, + display_app, trans.app, trans.request.base ) + + if len( display_links ) > 0: + display_label = hda.datatype.get_display_label( display_app ) + + app_links = [] + for display_name, display_link in display_links: + app_links.append({ + 'target': target_frame, + 'href' : display_link, + 'text' : gettext( display_name ) + }) + if app_links: + display_apps.append( dict( label=display_label, links=app_links ) ) + + return display_apps + + +# ============================================================================= +class HistorySerializer( manager_base.ModelSerializer ): + """ + Interface/service object for serializing HDAs into dictionaries. + """ + pass diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/managers/histories.py --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -1,9 +1,32 @@ +""" +Manager and Serializer for histories. + +Histories are containers for datasets or dataset collections +created (or copied) by users over the course of an analysis. +""" + from galaxy import exceptions from galaxy.model import orm +from galaxy.managers import base as manager_base +import galaxy.managers.hdas -class HistoryManager( object ): +import galaxy.web +import galaxy.dataset_collections.util + +import logging +log = logging.getLogger( __name__ ) + + +# ============================================================================= +class HistoryManager( manager_base.ModelManager ): + """ + Interface/service object for interacting with HDAs. + """ + #TODO: all the following would be more useful if passed the user instead of defaulting to trans.user + def __init__( self, *args, **kwargs ): + super( HistoryManager, self ).__init__( *args, **kwargs ) def get( self, trans, unencoded_id, check_ownership=True, check_accessible=True, deleted=None ): """ @@ -44,7 +67,7 @@ def secure( self, trans, history, check_ownership=True, check_accessible=True ): """ - checks if (a) user owns item or (b) item is accessible to user. + Checks if (a) user owns item or (b) item is accessible to user. """ # all items are accessible to an admin if trans.user and trans.user_is_admin(): @@ -56,15 +79,28 @@ return history def is_current( self, trans, history ): + """ + True if the given history is the user's current history. + + Returns False if the session has no current history. + """ + if trans.history is None: + return False return trans.history == history def is_owner( self, trans, history ): + """ + True if the current user is the owner of the given history. + """ # anon users are only allowed to view their current history if not trans.user: return self.is_current( trans, history ) return trans.user == history.user def check_ownership( self, trans, history ): + """ + Raises error if the current user is not the owner of the history. + """ if trans.user and trans.user_is_admin(): return history if not trans.user and not self.is_current( trans, history ): @@ -74,6 +110,9 @@ raise exceptions.ItemOwnershipException( "History is not owned by the current user", type='error' ) def is_accessible( self, trans, history ): + """ + True if the user can access (read) the current history. + """ # admin always have access if trans.user and trans.user_is_admin(): return True @@ -88,6 +127,103 @@ return False def check_accessible( self, trans, history ): + """ + Raises error if the current user can't access the history. + """ if self.is_accessible( trans, history ): return history raise exceptions.ItemAccessibilityException( "History is not accessible to the current user", type='error' ) + + #TODO: bleh... + def _get_history_data( self, trans, history ): + """ + Returns a dictionary containing ``history`` and ``contents``, serialized + history and an array of serialized history contents respectively. + """ + hda_mgr = galaxy.managers.hdas.HDAManager() + collection_dictifier = galaxy.dataset_collections.util.dictify_dataset_collection_instance + + history_dictionary = {} + contents_dictionaries = [] + try: + #for content in history.contents_iter( **contents_kwds ): + for content in history.contents_iter( types=[ 'dataset', 'dataset_collection' ] ): + hda_dict = {} + + if isinstance( content, trans.app.model.HistoryDatasetAssociation ): + try: + hda_dict = hda_mgr.get_hda_dict( trans, content ) + except Exception, exc: + # don't fail entire list if hda err's, record and move on + log.exception( 'Error bootstrapping hda: %s', exc ) + hda_dict = hda_mgr.get_hda_dict_with_error( trans, content, str( exc ) ) + + elif isinstance( content, trans.app.model.HistoryDatasetCollectionAssociation ): + try: + service = trans.app.dataset_collections_service + dataset_collection_instance = service.get_dataset_collection_instance( + trans=trans, + instance_type='history', + id=trans.security.encode_id( content.id ), + ) + hda_dict = collection_dictifier( dataset_collection_instance, + security=trans.security, parent=dataset_collection_instance.history, view="element" ) + + except Exception, exc: + log.exception( "Error in history API at listing dataset collection: %s", exc ) + #TODO: return some dict with the error + + contents_dictionaries.append( hda_dict ) + + # re-use the hdas above to get the history data... + history_dictionary = self.get_history_dict( trans, history, contents_dictionaries=contents_dictionaries ) + + except Exception, exc: + user_id = str( trans.user.id ) if trans.user else '(anonymous)' + log.exception( 'Error bootstrapping history for user %s: %s', user_id, str( exc ) ) + message = ( 'An error occurred getting the history data from the server. ' + + 'Please contact a Galaxy administrator if the problem persists.' ) + history_dictionary[ 'error' ] = message + + return { + 'history' : history_dictionary, + 'contents' : contents_dictionaries + } + + def get_history_dict( self, trans, history, contents_dictionaries=None ): + """ + Returns history data in the form of a dictionary. + """ + #TODO: to serializer + history_dict = history.to_dict( view='element', value_mapper={ 'id':trans.security.encode_id }) + history_dict[ 'user_id' ] = None + if history.user_id: + history_dict[ 'user_id' ] = trans.security.encode_id( history.user_id ) + + history_dict[ 'nice_size' ] = history.get_disk_size( nice_size=True ) + history_dict[ 'annotation' ] = history.get_item_annotation_str( trans.sa_session, history.user, history ) + if not history_dict[ 'annotation' ]: + history_dict[ 'annotation' ] = '' + + #TODO: item_slug url + if history_dict[ 'importable' ] and history_dict[ 'slug' ]: + username_and_slug = ( '/' ).join(( 'u', history.user.username, 'h', history_dict[ 'slug' ] )) + history_dict[ 'username_and_slug' ] = username_and_slug + +#TODO: re-add + #hda_summaries = hda_dictionaries if hda_dictionaries else self.get_hda_summary_dicts( trans, history ) + ##TODO remove the following in v2 + #( state_counts, state_ids ) = self._get_hda_state_summaries( trans, hda_summaries ) + #history_dict[ 'state_details' ] = state_counts + #history_dict[ 'state_ids' ] = state_ids + #history_dict[ 'state' ] = self._get_history_state_from_hdas( trans, history, state_counts ) + + return history_dict + + +# ============================================================================= +class HistorySerializer( manager_base.ModelSerializer ): + """ + Interface/service object for serializing histories into dictionaries. + """ + pass diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py --- a/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py +++ b/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py @@ -23,6 +23,8 @@ handler.setFormatter( formatter ) log.addHandler( handler ) +metadata = MetaData() +context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) class DeferredJob( object ): states = Bunch( NEW = 'new', @@ -37,12 +39,8 @@ self.params = params def upgrade(migrate_engine): - metadata = MetaData() metadata.bind = migrate_engine - Session = sessionmaker( bind=migrate_engine) - context = Session() - DeferredJob.table = Table( "deferred_job", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), @@ -70,12 +68,8 @@ context.flush() def downgrade(migrate_engine): - metadata = MetaData() metadata.bind = migrate_engine - Session = sessionmaker( bind=migrate_engine) - context = Session() - jobs = context.query( DeferredJob ).filter_by( plugin='GenomeTransferPlugin' ).all() for job in jobs: diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -3213,7 +3213,9 @@ # Populate tool_type to ToolClass mappings tool_types = {} -for tool_class in [ Tool, DataDestinationTool, SetMetadataTool, DataSourceTool, AsyncDataSourceTool, DataManagerTool ]: +for tool_class in [ Tool, SetMetadataTool, OutputParameterJSONTool, + DataManagerTool, DataSourceTool, AsyncDataSourceTool, + DataDestinationTool ]: tool_types[ tool_class.tool_type ] = tool_class diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/tools/actions/index_genome.py --- a/lib/galaxy/tools/actions/index_genome.py +++ /dev/null @@ -1,67 +0,0 @@ -import tempfile -from __init__ import ToolAction -from galaxy.util.odict import odict -from galaxy.tools.genome_index import * - -import logging -log = logging.getLogger( __name__ ) - -class GenomeIndexToolAction( ToolAction ): - """Tool action used for exporting a history to an archive. """ - - def execute( self, tool, trans, *args, **kwargs ): - # - # Get genome to index. - # - incoming = kwargs['incoming'] - # - # Create the job and output dataset objects - # - job = trans.app.model.Job() - job.tool_id = tool.id - job.user_id = incoming['user'] - start_job_state = job.state # should be job.states.NEW - job.state = job.states.WAITING # we need to set job state to something other than NEW, - # or else when tracking jobs in db it will be picked up - # before we have added input / output parameters - trans.sa_session.add( job ) - - # Create dataset that will serve as archive. - temp_dataset = trans.app.model.Dataset( state=trans.app.model.Dataset.states.NEW ) - trans.sa_session.add( temp_dataset ) - - trans.sa_session.flush() # ensure job.id and archive_dataset.id are available - trans.app.object_store.create( temp_dataset ) # set the object store id, create dataset (because galaxy likes having datasets) - - # - # Setup job and job wrapper. - # - - # Add association for keeping track of index jobs, transfer jobs, and so on. - user = trans.sa_session.query( trans.app.model.User ).get( int( incoming['user'] ) ) - assoc = trans.app.model.GenomeIndexToolData( job=job, dataset=temp_dataset, fasta_path=incoming['path'], \ - indexer=incoming['indexer'], user=user, \ - deferred_job=kwargs['deferred'], transfer_job=kwargs['transfer'] ) - trans.sa_session.add( assoc ) - - job_wrapper = GenomeIndexToolWrapper( job ) - cmd_line = job_wrapper.setup_job( assoc ) - - # - # Add parameters to job_parameter table. - # - incoming[ '__GENOME_INDEX_COMMAND__' ] = cmd_line - for name, value in tool.params_to_strings( incoming, trans.app ).iteritems(): - job.add_parameter( name, value ) - - job.state = start_job_state # job inputs have been configured, restore initial job state - job.set_handler(tool.get_job_handler(None)) - trans.sa_session.flush() - - - # Queue the job for execution - trans.app.job_queue.put( job.id, tool.id ) - log.info( "Added genome index job to the job queue, id: %s" % str( job.id ) ) - - return job, odict() - diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/tools/deps/containers.py --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -191,19 +191,26 @@ # TODO: Remove redundant volumes... volumes = docker_util.DockerVolume.volumes_from_str(volumes_raw) volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM) - return docker_util.build_docker_run_command( + + docker_host_props = dict( + docker_cmd=prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), + sudo=asbool(prop("sudo", docker_util.DEFAULT_SUDO)), + sudo_cmd=prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), + host=prop("host", docker_util.DEFAULT_HOST), + ) + + cache_command = docker_util.build_docker_cache_command(self.container_id, **docker_host_props) + run_command = docker_util.build_docker_run_command( command, self.container_id, volumes=volumes, volumes_from=volumes_from, env_directives=env_directives, working_directory=working_directory, - docker_cmd=prop("cmd", docker_util.DEFAULT_DOCKER_COMMAND), - sudo=asbool(prop("sudo", docker_util.DEFAULT_SUDO)), - sudo_cmd=prop("sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND), - host=prop("host", docker_util.DEFAULT_HOST), - net=prop("net", "none") # By default, docker instance has networking disabled + net=prop("net", "none"), # By default, docker instance has networking disabled + **docker_host_props ) + return "%s\n%s" % (cache_command, run_command) def __expand_str(self, value): if not value: diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/tools/deps/docker_util.py --- a/lib/galaxy/tools/deps/docker_util.py +++ b/lib/galaxy/tools/deps/docker_util.py @@ -50,28 +50,41 @@ return ":".join([self.from_path, self.to_path, self.how]) +def build_docker_cache_command( + image, + docker_cmd=DEFAULT_DOCKER_COMMAND, + sudo=DEFAULT_SUDO, + sudo_cmd=DEFAULT_SUDO_COMMAND, + host=DEFAULT_HOST, +): + inspect_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + inspect_command_parts.extend(["inspect", image]) + inspect_image_command = " ".join(inspect_command_parts) + + pull_command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) + pull_command_parts.extend(["pull", image]) + pull_image_command = " ".join(pull_command_parts) + cache_command = "%s > /dev/null 2>&1\n[ $? -ne 0 ] && %s > /dev/null 2>&1\n" % (inspect_image_command, pull_image_command) + return cache_command + + def build_docker_run_command( container_command, image, tag=None, - docker_cmd=DEFAULT_DOCKER_COMMAND, volumes=[], volumes_from=DEFAULT_VOLUMES_FROM, memory=DEFAULT_MEMORY, env_directives=[], working_directory=DEFAULT_WORKING_DIRECTORY, + name=None, + net=DEFAULT_NET, + docker_cmd=DEFAULT_DOCKER_COMMAND, sudo=DEFAULT_SUDO, sudo_cmd=DEFAULT_SUDO_COMMAND, - name=None, host=DEFAULT_HOST, - net=DEFAULT_NET, ): - command_parts = [] - if sudo: - command_parts.append(sudo_cmd) - command_parts.append(docker_cmd) - if host: - command_parts.append(["-H", host]) + command_parts = __docker_prefix(docker_cmd, sudo, sudo_cmd, host) command_parts.append("run") for env_directive in env_directives: command_parts.extend(["-e", env_directive]) @@ -93,3 +106,15 @@ command_parts.append(full_image) command_parts.append(container_command) return " ".join(command_parts) + + +def __docker_prefix(docker_cmd, sudo, sudo_cmd, host): + """ Prefix to issue a docker command. + """ + command_parts = [] + if sudo: + command_parts.append(sudo_cmd) + command_parts.append(docker_cmd) + if host: + command_parts.append(["-H", host]) + return command_parts diff -r 47c6c6bb8eec81060cbf2ddb2396c19e78444747 -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b lib/galaxy/tools/genome_index/__init__.py --- a/lib/galaxy/tools/genome_index/__init__.py +++ /dev/null @@ -1,243 +0,0 @@ -from __future__ import with_statement - -import json -import logging -import os -import shutil -import tarfile -import tempfile - -from galaxy import model, util -from galaxy.web.framework.helpers import to_unicode -from galaxy.model.item_attrs import UsesAnnotations -from galaxy.util.json import * -from galaxy.web.base.controller import UsesHistoryMixin -from galaxy.tools.data import ToolDataTableManager - - -log = logging.getLogger(__name__) - -def load_genome_index_tools( toolbox ): - """ Adds tools for indexing genomes via the main job runner. """ - # Create XML for loading the tool. - tool_xml_text = """ - <tool id="__GENOME_INDEX__" name="Index Genome" version="0.1" tool_type="genome_index"> - <type class="GenomeIndexTool" module="galaxy.tools"/> - <action module="galaxy.tools.actions.index_genome" class="GenomeIndexToolAction"/> - <command>$__GENOME_INDEX_COMMAND__ $output_file $output_file.files_path "$__app__.config.rsync_url" "$__app__.config.tool_data_path"</command> - <inputs> - <param name="__GENOME_INDEX_COMMAND__" type="hidden"/> - </inputs> - <outputs> - <data format="txt" name="output_file"/> - </outputs> - <stdio> - <exit_code range="1:" err_level="fatal" /> - </stdio> - </tool> - """ - - # Load index tool. - tmp_name = tempfile.NamedTemporaryFile() - tmp_name.write( tool_xml_text ) - tmp_name.flush() - genome_index_tool = toolbox.load_tool( tmp_name.name ) - toolbox.tools_by_id[ genome_index_tool.id ] = genome_index_tool - log.debug( "Loaded genome index tool: %s", genome_index_tool.id ) - -class GenomeIndexToolWrapper( object ): - """ Provides support for performing jobs that index a genome. """ - def __init__( self, job_id ): - self.locations = dict() - self.job_id = job_id - - def setup_job( self, genobj ): - """ Perform setup for job to index a genome and return an archive. Method generates - attribute files, sets the corresponding attributes in the associated database - object, and returns a command line for running the job. The command line - includes the command, inputs, and options; it does not include the output - file because it must be set at runtime. """ - - # - # Create and return command line for running tool. - # - scriptpath = os.path.join( os.path.abspath( os.getcwd() ), "lib/galaxy/tools/genome_index/index_genome.py" ) - return "python %s %s %s" % ( scriptpath, genobj.indexer, genobj.fasta_path ) - - def postprocessing( self, sa_session, app ): - """ Finish the job, move the finished indexes to their final resting place, - and update the .loc files where applicable. """ - gitd = sa_session.query( model.GenomeIndexToolData ).filter_by( job_id=self.job_id ).first() - indexdirs = dict( bfast='bfast_index', bowtie='bowtie_index', bowtie2='bowtie2_index', - bwa='bwa_index', perm='perm_%s_index', picard='srma_index', sam='sam_index' ) - - - if gitd: - fp = open( gitd.dataset.get_file_name(), 'r' ) - deferred = sa_session.query( model.DeferredJob ).filter_by( id=gitd.deferred_job_id ).first() - try: - logloc = json.load( fp ) - except ValueError: - deferred.state = app.model.DeferredJob.states.ERROR - sa_session.add( deferred ) - sa_session.flush() - log.debug( 'Indexing job failed, setting deferred job state to error.' ) - return False - finally: - fp.close() - destination = None - tdtman = ToolDataTableManager( app.config.tool_data_path ) - xmltree = tdtman.load_from_config_file( app.config.tool_data_table_config_path, app.config.tool_data_path ) - for node in xmltree: - table = node.get('name') - location = node.findall('file')[0].get('path') - self.locations[table] = os.path.abspath( location ) - locbase = os.path.abspath( os.path.split( self.locations['all_fasta'] )[0] ) - params = deferred.params - dbkey = params[ 'dbkey' ] - basepath = os.path.join( os.path.abspath( app.config.genome_data_path ), dbkey ) - intname = params[ 'intname' ] - indexer = gitd.indexer - workingdir = os.path.abspath( gitd.dataset.extra_files_path ) - location = [] - indexdata = gitd.dataset.extra_files_path - if indexer == '2bit': - indexdata = os.path.join( workingdir, '%s.2bit' % dbkey ) - destination = os.path.join( basepath, 'seq', '%s.2bit' % dbkey ) - location.append( dict( line='\t'.join( [ 'seq', dbkey, destination ] ), file= os.path.join( locbase, 'alignseq.loc' ) ) ) - elif indexer == 'bowtie': - self._ex_tar( workingdir, 'cs.tar' ) - destination = os.path.join( basepath, 'bowtie_index' ) - for var in [ 'nt', 'cs' ]: - for line in logloc[ var ]: - idx = line - if var == 'nt': - locfile = self.locations[ 'bowtie_indexes' ] - locdir = os.path.join( destination, idx ) - else: - locfile = self.locations[ 'bowtie_indexes_color' ] - locdir = os.path.join( destination, var, idx ) - location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) - elif indexer == 'bowtie2': - destination = os.path.join( basepath, 'bowtie2_index' ) - for line in logloc[ 'nt' ]: - idx = line - locfile = self.locations[ 'bowtie2_indexes' ] - locdir = os.path.join( destination, idx ) - location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) - elif indexer == 'bwa': - self._ex_tar( workingdir, 'cs.tar' ) - destination = os.path.join( basepath, 'bwa_index' ) - for var in [ 'nt', 'cs' ]: - for line in logloc[ var ]: - idx = line - if var == 'nt': - locfile = self.locations[ 'bwa_indexes' ] - locdir = os.path.join( destination, idx ) - else: - locfile = self.locations[ 'bwa_indexes_color' ] - locdir = os.path.join( destination, var, idx ) - location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) - elif indexer == 'perm': - self._ex_tar( workingdir, 'cs.tar' ) - destination = os.path.join( basepath, 'perm_index' ) - for var in [ 'nt', 'cs' ]: - for line in logloc[ var ]: - idx = line.pop() - if var == 'nt': - locfile = self.locations[ 'perm_base_indexes' ] - locdir = os.path.join( destination, idx ) - else: - locfile = self.locations[ 'perm_color_indexes' ] - locdir = os.path.join( destination, var, idx ) - line.append( locdir ) - location.append( dict( line='\t'.join( line ), file=locfile ) ) - elif indexer == 'picard': - destination = os.path.join( basepath, 'srma_index' ) - for var in [ 'nt' ]: - for line in logloc[ var ]: - idx = line - locfile = self.locations[ 'picard_indexes' ] - locdir = os.path.join( destination, idx ) - location.append( dict( line='\t'.join( [ dbkey, dbkey, intname, locdir ] ), file=locfile ) ) - elif indexer == 'sam': - destination = os.path.join( basepath, 'sam_index' ) - for var in [ 'nt' ]: - for line in logloc[ var ]: - locfile = self.locations[ 'sam_fa_indexes' ] - locdir = os.path.join( destination, line ) - location.append( dict( line='\t'.join( [ 'index', dbkey, locdir ] ), file=locfile ) ) - - if destination is not None and os.path.exists( os.path.split( destination )[0] ) and not os.path.exists( destination ): - log.debug( 'Moving %s to %s' % ( indexdata, destination ) ) - shutil.move( indexdata, destination ) - if indexer not in [ '2bit' ]: - genome = '%s.fa' % dbkey - target = os.path.join( destination, genome ) - fasta = os.path.abspath( os.path.join( basepath, 'seq', genome ) ) - self._check_link( fasta, target ) - if os.path.exists( os.path.join( destination, 'cs' ) ): - target = os.path.join( destination, 'cs', genome ) - fasta = os.path.abspath( os.path.join( basepath, 'seq', genome ) ) - self._check_link( fasta, target ) - for line in location: - self._add_line( line[ 'file' ], line[ 'line' ] ) - deferred.state = app.model.DeferredJob.states.OK - sa_session.add( deferred ) - sa_session.flush() - - - def _check_link( self, targetfile, symlink ): - target = os.path.relpath( targetfile, os.path.dirname( symlink ) ) - filename = os.path.basename( targetfile ) - if not os.path.exists( targetfile ): # this should never happen. - raise Exception, "%s not found. Unable to proceed without a FASTA file. Aborting." % targetfile - if os.path.exists( symlink ) and os.path.islink( symlink ): - if os.path.realpath( symlink ) == os.path.abspath( targetfile ): # symlink exists, points to the correct FASTA file. - return - else: # no it doesn't. Make a new one, and this time do it right. - os.remove( symlink ) - os.symlink( target, symlink ) - return - elif not os.path.exists( symlink ): # no symlink to the FASTA file. Create one. - os.symlink( target, symlink ) - return - elif os.path.exists( symlink ) and not os.path.islink( symlink ): - if self._hash_file( targetfile ) == self._hash_file( symlink ): # files are identical. No need to panic. - return - else: - if os.path.getsize( symlink ) == 0: # somehow an empty file got copied instead of the symlink. Delete with extreme prejudice. - os.remove( symlink ) - os.symlink( target, symlink ) - return - else: - raise Exception, "Regular file %s exists, is not empty, contents do not match %s." % ( symlink, targetfile ) - - def _hash_file( self, filename ): - import hashlib - md5 = hashlib.md5() - with open( filename, 'rb' ) as f: - for chunk in iter( lambda: f.read( 8192 ), '' ): - md5.update( chunk ) - return md5.digest() - - - def _ex_tar( self, directory, filename ): - fh = tarfile.open( os.path.join( directory, filename ) ) - fh.extractall( path=directory ) - fh.close() - os.remove( os.path.join( directory, filename ) ) - - def _add_line( self, locfile, newline ): - filepath = locfile - origlines = [] - output = [] - comments = [] - with open( filepath, 'r' ) as destfile: - for line in destfile: - origlines.append( line.strip() ) - if newline not in origlines: - origlines.append( newline ) - with open( filepath, 'w+' ) as destfile: - origlines.append( '' ) - destfile.write( '\n'.join( origlines ) ) This diff is so big that we needed to truncate the remainder. https://bitbucket.org/galaxy/galaxy-central/commits/4ca1677c8d5b/ Changeset: 4ca1677c8d5b User: iracooke Date: 2014-07-24 09:16:24 Summary: Added whitelisting to sqlite data provider Affected #: 1 file diff -r 034c0159c0cfbc7736eeee4132fbedda8a465c6b -r 4ca1677c8d5bc6486b0b1dff50372a23dfaf307d lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -12,6 +12,7 @@ import column import external import sqlite3 +import re from galaxy import eggs eggs.require( 'bx-python' ) @@ -720,8 +721,18 @@ self.connection.row_factory = sqlite3.Row super( SQliteDataProvider, self ).__init__( source, **kwargs ) + def query_matches_whitelist(self,query): + if re.match("select ",query,re.IGNORECASE): + if re.search("^([^\"]|\"[^\"]*\")*?;",query) or re.search("^([^\']|\'[^\']*\')*?;",query): + return False + else: + return True + return False + + + def __iter__( self ): - if self.query is not None: + if (self.query is not None) and self.query_matches_whitelist(self.query): for row in self.connection.cursor().execute(self.query): yield row else: https://bitbucket.org/galaxy/galaxy-central/commits/56a7b27577de/ Changeset: 56a7b27577de User: dannon Date: 2014-07-25 00:29:57 Summary: Merged in iracooke/galaxy-central (pull request #434) Add sqlite datatype and corresponding dataprovider Affected #: 3 files diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 datatypes_conf.xml.sample --- a/datatypes_conf.xml.sample +++ b/datatypes_conf.xml.sample @@ -177,6 +177,7 @@ <datatype extension="taxonomy" type="galaxy.datatypes.tabular:Taxonomy" display_in_upload="true"/><datatype extension="tabular" type="galaxy.datatypes.tabular:Tabular" display_in_upload="true" description="Any data in tab delimited format (tabular)." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Tabular_.28tab_delimited.29"/><datatype extension="twobit" type="galaxy.datatypes.binary:TwoBit" mimetype="application/octet-stream" display_in_upload="true"/> + <datatype extension="sqlite" type="galaxy.datatypes.binary:SQlite" mimetype="application/octet-stream" display_in_upload="true"/><datatype extension="txt" type="galaxy.datatypes.data:Text" display_in_upload="true" description="Any text file." description_url="https://wiki.galaxyproject.org/Learn/Datatypes#Plain_text"/><datatype extension="linecount" type="galaxy.datatypes.data:LineCount" display_in_upload="false"/><datatype extension="memexml" type="galaxy.datatypes.xml:MEMEXml" mimetype="application/xml" display_in_upload="true"/> @@ -262,6 +263,7 @@ --><sniffer type="galaxy.datatypes.tabular:Vcf"/><sniffer type="galaxy.datatypes.binary:TwoBit"/> + <sniffer type="galaxy.datatypes.binary:SQlite"/><sniffer type="galaxy.datatypes.binary:Bam"/><sniffer type="galaxy.datatypes.binary:Sff"/><sniffer type="galaxy.datatypes.xml:Phyloxml"/> diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 lib/galaxy/datatypes/binary.py --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -12,6 +12,7 @@ import subprocess import tempfile import zipfile +import sqlite3 from urllib import urlencode, quote_plus from galaxy import eggs @@ -545,3 +546,45 @@ return "Binary TwoBit format nucleotide file (%s)" % (data.nice_size(dataset.get_size())) Binary.register_sniffable_binary_format("twobit", "twobit", TwoBit) + + +(a)dataproviders.decorators.has_dataproviders +class SQlite ( Binary ): + file_ext = "sqlite" + + # Connects and runs a query that should work on any real database + # If the file is not sqlite, an exception will be thrown and the sniffer will return false + def sniff( self, filename ): + try: + conn = sqlite3.connect(filename) + schema_version=conn.cursor().execute("pragma schema_version").fetchone() + conn.close() + if schema_version is not None: + return True + return False + except: + return False + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = "SQLite Database" + dataset.blurb = data.nice_size( dataset.get_size() ) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "SQLite Database (%s)" % ( data.nice_size( dataset.get_size() ) ) + + + @dataproviders.decorators.dataprovider_factory( 'sqlite', dataproviders.dataset.SQliteDataProvider.settings ) + def sqlite_dataprovider( self, dataset, **settings ): + dataset_source = dataproviders.dataset.DatasetDataProvider( dataset ) + return dataproviders.dataset.SQliteDataProvider( dataset_source, **settings ) + + +Binary.register_sniffable_binary_format("sqlite","sqlite",SQlite) + diff -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 -r 56a7b27577dec0dac17ef94d4c131e7f1f61cb15 lib/galaxy/datatypes/dataproviders/dataset.py --- a/lib/galaxy/datatypes/dataproviders/dataset.py +++ b/lib/galaxy/datatypes/dataproviders/dataset.py @@ -11,6 +11,8 @@ import line import column import external +import sqlite3 +import re from galaxy import eggs eggs.require( 'bx-python' ) @@ -700,3 +702,39 @@ #TODO: as samtools - need more info on output format raise NotImplementedError() super( BGzipTabixDataProvider, self ).__init__( dataset, **kwargs ) + + + +class SQliteDataProvider ( base.DataProvider ): + """ + Data provider that uses a sqlite database file as its source. + + Allows any query to be run and returns the resulting rows as sqlite3 row objects + """ + settings = { + 'query' : 'str' + } + + def __init__( self, source, query=None, **kwargs ): + self.query=query + self.connection = sqlite3.connect(source.dataset.file_name); + self.connection.row_factory = sqlite3.Row + super( SQliteDataProvider, self ).__init__( source, **kwargs ) + + def query_matches_whitelist(self,query): + if re.match("select ",query,re.IGNORECASE): + if re.search("^([^\"]|\"[^\"]*\")*?;",query) or re.search("^([^\']|\'[^\']*\')*?;",query): + return False + else: + return True + return False + + + + def __iter__( self ): + if (self.query is not None) and self.query_matches_whitelist(self.query): + for row in self.connection.cursor().execute(self.query): + yield row + else: + yield + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 7 new changesets
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

7 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e80c212ba514/ Changeset: e80c212ba514 User: dannon Date: 2014-07-23 23:24:33 Summary: Strip whitespace in manage_info template. Affected #: 1 file diff -r 6fbe4d95a8dc64bd222dbf1170bdadcc5321e855 -r e80c212ba514b4960c01f681cf15a2bc8b2b4882 templates/webapps/galaxy/user/manage_info.mako --- a/templates/webapps/galaxy/user/manage_info.mako +++ b/templates/webapps/galaxy/user/manage_info.mako @@ -49,7 +49,7 @@ <div class="form-row"><div class="grid-header"> %for i, filter in enumerate( ['Active', 'Deleted', 'All'] ): - %if i > 0: + %if i > 0: <span>|</span> %endif %if show_filter == filter: @@ -62,10 +62,10 @@ </div><table class="grid"><tbody> - %for index, address in enumerate(addresses): + %for index, address in enumerate(addresses): <tr class="libraryRow libraryOrFolderRow" id="libraryRow"><td> - <div class="form-row"> + <div class="form-row"><label>${address.desc}:</label> ${address.get_html()} </div> @@ -82,7 +82,7 @@ </ul></div></td> - </tr> + </tr> %endfor </tbody></table> https://bitbucket.org/galaxy/galaxy-central/commits/65f2c8c30d47/ Changeset: 65f2c8c30d47 User: dannon Date: 2014-07-24 22:04:39 Summary: Sanitize user addresses. This should be revisited at some point -- not sure why these html stubs (and only these) generated in the model. Affected #: 1 file diff -r e80c212ba514b4960c01f681cf15a2bc8b2b4882 -r 65f2c8c30d477eb1008bf4e3b40c8695f5ac2df6 lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -34,6 +34,7 @@ from galaxy.util.bunch import Bunch from galaxy.util.hash_util import new_secure_hash from galaxy.util.directory_hash import directory_hash_id +from galaxy.util.sanitize_html import sanitize_html from galaxy.web.framework.helpers import to_unicode from galaxy.web.form_builder import (AddressField, CheckboxField, HistoryField, PasswordField, SelectField, TextArea, TextField, WorkflowField, @@ -3703,23 +3704,27 @@ self.country = country self.phone = phone def get_html(self): + # This should probably be deprecated eventually. It should currently + # sanitize. + # TODO Find out where else uses this and replace with + # templates html = '' if self.name: - html = html + self.name + html = html + sanitize_html(self.name) if self.institution: - html = html + '<br/>' + self.institution + html = html + '<br/>' + sanitize_html(self.institution) if self.address: - html = html + '<br/>' + self.address + html = html + '<br/>' + sanitize_html(self.address) if self.city: - html = html + '<br/>' + self.city + html = html + '<br/>' + sanitize_html(self.city) if self.state: - html = html + ' ' + self.state + html = html + ' ' + sanitize_html(self.state) if self.postal_code: - html = html + ' ' + self.postal_code + html = html + ' ' + sanitize_html(self.postal_code) if self.country: - html = html + '<br/>' + self.country + html = html + '<br/>' + sanitize_html(self.country) if self.phone: - html = html + '<br/>' + 'Phone: ' + self.phone + html = html + '<br/>' + 'phone: ' + sanitize_html(self.phone) return html class UserOpenID( object ): https://bitbucket.org/galaxy/galaxy-central/commits/e39c3a510cc1/ Changeset: e39c3a510cc1 User: dannon Date: 2014-07-24 22:06:21 Summary: Sanitize address description display in the manage_info field. Affected #: 1 file diff -r 65f2c8c30d477eb1008bf4e3b40c8695f5ac2df6 -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 templates/webapps/galaxy/user/manage_info.mako --- a/templates/webapps/galaxy/user/manage_info.mako +++ b/templates/webapps/galaxy/user/manage_info.mako @@ -66,7 +66,7 @@ <tr class="libraryRow libraryOrFolderRow" id="libraryRow"><td><div class="form-row"> - <label>${address.desc}:</label> + <label>${address.desc | h}:</label> ${address.get_html()} </div><div class="form-row"> https://bitbucket.org/galaxy/galaxy-central/commits/6e6e184ec4e1/ Changeset: 6e6e184ec4e1 User: dannon Date: 2014-07-24 22:06:37 Summary: Merge. Affected #: 6 files diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1712,10 +1712,15 @@ if self.__display_multirun_option(): # Select multiple datasets, run multiple jobs. multirun_key = "%s|__multirun__" % self.name + collection_multirun_key = "%s|__collection_multirun__" % self.name if multirun_key in (other_values or {}): multirun_value = listify( other_values[ multirun_key ] ) if multirun_value and len( multirun_value ) > 1: default_field = "select_multiple" + elif collection_multirun_key in (other_values or {}): + multirun_value = listify( other_values[ collection_multirun_key ] ) + if multirun_value: + default_field = "select_collection" else: multirun_value = value multi_dataset_matcher = DatasetMatcher( trans, self, multirun_value, other_values ) @@ -2014,9 +2019,17 @@ default_field = "select_single_collection" fields = odict() + collection_multirun_key = "%s|__collection_multirun__" % self.name + if collection_multirun_key in (other_values or {}): + multirun_value = other_values[ collection_multirun_key ] + if multirun_value: + default_field = "select_map_over_collections" + else: + multirun_value = value + history = self._get_history( trans ) fields[ "select_single_collection" ] = self._get_single_collection_field( trans=trans, history=history, value=value, other_values=other_values ) - fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=value, other_values=other_values ) + fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=multirun_value, other_values=other_values ) return self._switch_fields( fields, default_field=default_field ) diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -86,7 +86,7 @@ }).attr( 'title', selectionType['select_by'] - ); + ).data( "index", iIndex ); view.formRow().find( "label" ).append( button ); } }); @@ -114,11 +114,13 @@ } else { $("div#remap-row").css("display", "none"); } - this.formRow().find( "i" ).each(function(index, iElement) { + this.formRow().find( "i" ).each(function(_, iElement) { + var $iElement = $(iElement); + var index = $iElement.data("index"); if(index == enableIndex) { - $(iElement).css('color', 'black'); + $iElement.css('color', 'black'); } else { - $(iElement).css('color', 'Gray'); + $iElement.css('color', 'Gray'); } }); var $select = this.$( "select" ); diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae static/scripts/viz/trackster.js --- a/static/scripts/viz/trackster.js +++ b/static/scripts/viz/trackster.js @@ -103,8 +103,9 @@ var self = this, menu = create_icon_buttons_menu([ { icon_class: 'plus-button', title: 'Add tracks', on_click: function() { - visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, function(tracks) { - _.each(tracks, function(track) { + visualization.select_datasets(galaxy_config.root + "visualization/list_current_history_datasets", galaxy_config.root + "api/datasets", { 'f-dbkey': view.dbkey }, + function(new_tracks) { + _.each(new_tracks, function(track) { view.add_drawable( tracks.object_from_template(track, view, view) ); }); }); diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/functional/tools/collection_two_paired.xml --- /dev/null +++ b/test/functional/tools/collection_two_paired.xml @@ -0,0 +1,69 @@ +<tool id="collection_two_paired" name="collection_two_paired" version="0.1.0"> + <command> + #if $kind.collection_type == "paired" + cat $kind.f1.forward $kind.f1['reverse'] >> $out1; + cat $kind.f2.forward $kind.f2['reverse'] >> $out1; + #else + #for $i, $_ in enumerate($kind.f1): + cat $kind.f1[$i] $kind.f2[$i] >> $out1; + #end for + #end if + </command> + <inputs> + <conditional name="kind"> + <param type="select" name="collection_type"> + <option value="paired">Paired Datasets</option> + <option value="list">List of Datasets</option> + </param> + <when value="paired"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + <when value="list"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="out1" /> + </outputs> + <tests> + <test> + <conditional name="kind"> + <param name="collection_type" value="paired" /> + <param name="f1"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + <test> + <conditional name="kind"> + <param name="collection_type" value="list" /> + <param name="f1"> + <collection type="list"> + <element name="l11" value="simple_line.txt" /> + <element name="l12" value="simple_line.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="list"> + <element name="l21" value="simple_line_alternative.txt" /> + <element name="l22" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -21,4 +21,5 @@ <tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> + <tool file="collection_two_paired.xml" /></toolbox> \ No newline at end of file diff -r e39c3a510cc17b47f3e54b0dcdc1965be8287be8 -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae test/unit/tools/test_execution.py --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -16,13 +16,12 @@ eggs.require( "Paste" ) from paste import httpexceptions -# Tool with a repeat parameter, to test state update. -REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"><command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="text" name="param1" value="" /><repeat name="repeat1" label="Repeat 1"> - <param type="text" name="param2" value="" /> + %s </repeat></inputs><outputs> @@ -31,6 +30,10 @@ </tool> ''' +# Tool with a repeat parameter, to test state update. +REPEAT_TOOL_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="text" name="param2" value="" />''' +REPEAT_COLLECTION_PARAM_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="data_collection" name="param2" collection_type="paired" />''' + class ToolExecutionTestCase( TestCase, tools_support.UsesApp, tools_support.UsesTools ): @@ -287,13 +290,48 @@ } ) self.__assert_exeuted( template, template_vars ) - def __history_dataset_collection_for( self, hdas, id=1234 ): - collection = galaxy.model.DatasetCollection() + def test_subcollection_multirun_with_state_updates( self ): + self._init_tool( REPEAT_COLLECTION_PARAM_CONTENTS ) + hda1, hda2 = self.__add_dataset( 1 ), self.__add_dataset( 2 ) + collection = self.__history_dataset_collection_for( [ hda1, hda2 ], collection_type="list:paired" ) + collection_id = self.app.security.encode_id( collection.id ) + self.app.dataset_collections_service = Bunch( + match_collections=lambda collections: None + ) + template, template_vars = self.__handle_with_incoming( + repeat1_add="dummy", + ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert len( state.inputs[ "repeat1" ] ) == 1 + template, template_vars = self.__handle_with_incoming( state, **{ + "repeat1_0|param2|__collection_multirun__": "%s|paired" % collection_id, + "repeat1_add": "dummy", + } ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert state.inputs[ "repeat1" ][ 0 ][ "param2|__collection_multirun__" ] == "%s|paired" % collection_id + + def __history_dataset_collection_for( self, hdas, collection_type="list", id=1234 ): + collection = galaxy.model.DatasetCollection( + collection_type=collection_type, + ) to_element = lambda hda: galaxy.model.DatasetCollectionElement( collection=collection, element=hda, ) - collection.datasets = map(to_element, hdas) + elements = map(to_element, hdas) + if collection_type == "list:paired": + paired_collection = galaxy.model.DatasetCollection( + collection_type="paired", + ) + paired_collection.elements = elements + list_dce = galaxy.model.DatasetCollectionElement( + collection=collection, + element=paired_collection, + ) + elements = [ list_dce ] + + collection.elements = elements + history_dataset_collection_association = galaxy.model.HistoryDatasetCollectionAssociation( id=id, collection=collection, @@ -349,13 +387,13 @@ self.history.datasets.append( hda ) return hda - def __add_collection_dataset( self, id, *hdas ): + def __add_collection_dataset( self, id, collection_type="paired", *hdas ): hdca = galaxy.model.HistoryDatasetCollectionAssociation() hdca.id = id collection = galaxy.model.DatasetCollection() hdca.collection = collection collection.elements = [ galaxy.model.DatasetCollectionElement(element=self.__add_dataset( 1 )) ] - + collection.type = collection_type self.trans.sa_session.model_objects[ galaxy.model.HistoryDatasetCollectionAssociation ][ id ] = hdca self.history.dataset_collections.append( hdca ) return hdca https://bitbucket.org/galaxy/galaxy-central/commits/3686de5c8280/ Changeset: 3686de5c8280 User: dannon Date: 2014-07-24 23:29:51 Summary: merge Affected #: 2 files diff -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae -r 3686de5c8280c18c197323948c5e2ea79e642a0b lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1391,7 +1391,7 @@ # Load parameters (optional) input_elem = root.find("inputs") enctypes = set() - if input_elem: + if input_elem is not None: # Handle properties of the input form self.check_values = string_as_bool( input_elem.get("check_values", self.check_values ) ) self.nginx_upload = string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) ) diff -r 6e6e184ec4e1e00ff12b23ec36c9b7d3671e19ae -r 3686de5c8280c18c197323948c5e2ea79e642a0b test/functional/tools/collection_two_paired.xml --- a/test/functional/tools/collection_two_paired.xml +++ b/test/functional/tools/collection_two_paired.xml @@ -20,8 +20,8 @@ <param name="f2" type="data_collection" collection_type="paired" /></when><when value="list"> - <param name="f1" type="data_collection" collection_type="paired" /> - <param name="f2" type="data_collection" collection_type="paired" /> + <param name="f1" type="data_collection" collection_type="list" /> + <param name="f2" type="data_collection" collection_type="list" /></when></conditional></inputs> https://bitbucket.org/galaxy/galaxy-central/commits/dd39f52ba6fc/ Changeset: dd39f52ba6fc User: dannon Date: 2014-07-24 23:52:30 Summary: Cleanup in galaxy.util Affected #: 1 file diff -r 3686de5c8280c18c197323948c5e2ea79e642a0b -r dd39f52ba6fc0cea9328d1437842904ac1d020c9 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -48,10 +48,10 @@ from .inflection import Inflector, English inflector = Inflector(English) -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) _lock = threading.RLock() -CHUNK_SIZE = 65536 #64k +CHUNK_SIZE = 65536 # 64k DATABASE_MAX_STRING_SIZE = 32768 DATABASE_MAX_STRING_SIZE_PRETTY = '32K' @@ -62,6 +62,7 @@ NULL_CHAR = '\000' BINARY_CHARS = [ NULL_CHAR ] + def is_multi_byte( chars ): for char in chars: try: @@ -69,18 +70,15 @@ except UnicodeDecodeError: # Probably binary return False - if wchartype.is_asian( char ) or \ - wchartype.is_full_width( char ) or \ - wchartype.is_kanji( char ) or \ - wchartype.is_hiragana( char ) or \ - wchartype.is_katakana( char ) or \ - wchartype.is_half_katakana( char ) or \ - wchartype.is_hangul( char ) or \ - wchartype.is_full_digit( char ) or \ - wchartype.is_full_letter( char ): + if ( wchartype.is_asian( char ) or wchartype.is_full_width( char ) or + wchartype.is_kanji( char ) or wchartype.is_hiragana( char ) or + wchartype.is_katakana( char ) or wchartype.is_half_katakana( char ) + or wchartype.is_hangul( char ) or wchartype.is_full_digit( char ) + or wchartype.is_full_letter( char )): return True return False + def is_binary( value, binary_chars=None ): """ File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.). @@ -99,6 +97,7 @@ return True return False + def get_charset_from_http_headers( headers, default=None ): rval = headers.get('content-type', None ) if rval and 'charset=' in rval: @@ -107,16 +106,18 @@ return rval return default + def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" def caller(*params, **kparams): - _lock.acquire(True) # Wait + _lock.acquire(True) # Wait try: return func(*params, **kparams) finally: _lock.release() return caller + def file_iter(fname, sep=None): """ This generator iterates over a file and yields its lines @@ -131,6 +132,7 @@ if line and line[0] != '#': yield line.split(sep) + def file_reader( fp, chunk_size=CHUNK_SIZE ): """This generator yields the open fileobject in chunks (default 64k). Closes the file at the end""" while 1: @@ -140,6 +142,7 @@ yield data fp.close() + def unique_id(KEY_SIZE=128): """ Generates an unique id @@ -148,8 +151,8 @@ >>> len(set(ids)) 1000 """ - id = str( random.getrandbits( KEY_SIZE ) ) - return md5(id).hexdigest() + return md5(str( random.getrandbits( KEY_SIZE ) )).hexdigest() + def parse_xml(fname): """Returns a parsed xml tree""" @@ -163,6 +166,7 @@ tree = ElementTree.fromstring(xml_string) return tree + def xml_to_string( elem, pretty=False ): """Returns a string from an xml tree""" if pretty: @@ -170,12 +174,13 @@ try: return ElementTree.tostring( elem ) except TypeError, e: - #assume this is a comment + # we assume this is a comment if hasattr( elem, 'text' ): return "\n" % ( elem.text ) else: raise e + def xml_element_compare( elem1, elem2 ): if not isinstance( elem1, dict ): elem1 = xml_element_to_dict( elem1 ) @@ -183,9 +188,11 @@ elem2 = xml_element_to_dict( elem2 ) return elem1 == elem2 + def xml_element_list_compare( elem_list1, elem_list2 ): return [ xml_element_to_dict( elem ) for elem in elem_list1 ] == [ xml_element_to_dict( elem ) for elem in elem_list2 ] + def xml_element_to_dict( elem ): rval = {} if elem.attrib: @@ -220,7 +227,6 @@ return rval - def pretty_print_xml( elem, level=0 ): pad = ' ' i = "\n" + level * pad @@ -238,26 +244,28 @@ elem.tail = i + pad return elem + def get_file_size( value, default=None ): try: - #try built-in + # try built-in return os.path.getsize( value ) except: try: - #try built-in one name attribute + # try built-in one name attribute return os.path.getsize( value.name ) except: try: - #try tell() of end of object + # try tell() of end of object offset = value.tell() value.seek( 0, 2 ) rval = value.tell() value.seek( offset ) return rval except: - #return default value + # return default value return default + def shrink_stream_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): rval = '' if get_file_size( value ) > size: @@ -292,6 +300,7 @@ rval += data return rval + def shrink_string_by_size( value, size, join_by="..", left_larger=True, beginning_on_size_error=False, end_on_size_error=False ): if len( value ) > size: len_join_by = len( join_by ) @@ -311,29 +320,30 @@ value = "%s%s%s" % ( value[:left_index], join_by, value[-right_index:] ) return value + def pretty_print_json(json_data, is_json_string=False): if is_json_string: json_data = json.from_json_string(json_data) return json.to_json_string(json_data, sort_keys=True, indent=4) # characters that are valid -valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") +valid_chars = set(string.letters + string.digits + " -=_.()/+*^,:?!") # characters that are allowed but need to be escaped -mapped_chars = { '>' :'__gt__', - '<' :'__lt__', - "'" :'__sq__', - '"' :'__dq__', - '[' :'__ob__', - ']' :'__cb__', - '{' :'__oc__', - '}' :'__cc__', - '@' : '__at__', - '\n' : '__cn__', - '\r' : '__cr__', - '\t' : '__tc__', - '#' : '__pd__' - } +mapped_chars = { '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '@': '__at__', + '\n': '__cn__', + '\r': '__cr__', + '\t': '__tc__', + '#': '__pd__'} + def restore_text(text): """Restores sanitized text""" @@ -343,6 +353,7 @@ text = text.replace(value, key) return text + def sanitize_text(text): """ Restricts the characters that are allowed in text; accepts both strings @@ -353,6 +364,7 @@ elif isinstance( text, list ): return [ _sanitize_text_helper(t) for t in text ] + def _sanitize_text_helper(text): """Restricts the characters that are allowed in a string""" @@ -363,9 +375,10 @@ elif c in mapped_chars: out.append(mapped_chars[c]) else: - out.append('X') # makes debugging easier + out.append('X') # makes debugging easier return ''.join(out) + def sanitize_param(value): """Clean incoming parameters (strings or lists)""" if isinstance( value, basestring ): @@ -373,10 +386,12 @@ elif isinstance( value, list ): return map(sanitize_text, value) else: - raise Exception, 'Unknown parameter type (%s)' % ( type( value ) ) + raise Exception('Unknown parameter type (%s)' % ( type( value ) )) valid_filename_chars = set( string.ascii_letters + string.digits + '_.' ) invalid_filenames = [ '', '.', '..' ] + + def sanitize_for_filename( text, default=None ): """ Restricts the characters that are allowed in a filename portion; Returns default value or a unique id string if result is not a valid name. @@ -512,7 +527,7 @@ def __init__( self, params, sanitize=True ): if sanitize: for key, value in params.items(): - if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: #sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. + if key not in self.NEVER_SANITIZE and True not in [ key.endswith( "|%s" % nonsanitize_parameter ) for nonsanitize_parameter in self.NEVER_SANITIZE ]: # sanitize check both ungrouped and grouped parameters by name. Anything relying on NEVER_SANITIZE should be changed to not require this and NEVER_SANITIZE should be removed. self.__dict__[ key ] = sanitize_param( value ) else: self.__dict__[ key ] = value @@ -525,7 +540,7 @@ """ flat = [] for key, value in self.__dict__.items(): - if type(value) == type([]): + if isinstance(value, list): for v in value: flat.append( (key, v) ) else: @@ -551,16 +566,19 @@ def update(self, values): self.__dict__.update(values) + def rst_to_html( s ): """Convert a blob of reStructuredText to HTML""" log = logging.getLogger( "docutils" ) + class FakeStream( object ): def write( self, str ): if len( str ) > 0 and not str.isspace(): log.warn( str ) return unicodify( docutils.core.publish_string( s, - writer=docutils.writers.html4css1.Writer(), - settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + writer=docutils.writers.html4css1.Writer(), + settings_overrides={ "embed_stylesheet": False, "template": os.path.join(os.path.dirname(__file__), "docutils_template.txt"), "warning_stream": FakeStream() } ) ) + def xml_text(root, name=None): """Returns the text inside an element""" @@ -582,6 +600,8 @@ # asbool implementation pulled from PasteDeploy truthy = frozenset(['true', 'yes', 'on', 'y', 't', '1']) falsy = frozenset(['false', 'no', 'off', 'n', 'f', '0']) + + def asbool(obj): if isinstance(obj, basestring): obj = obj.strip().lower() @@ -600,6 +620,7 @@ else: return False + def string_as_bool_or_none( string ): """ Returns True, None or False based on the argument: @@ -618,6 +639,7 @@ else: return False + def listify( item, do_strip=False ): """ Make a single item a single item list, or return a list if passed a @@ -635,6 +657,7 @@ else: return [ item ] + def commaify(amount): orig = amount new = re.sub("^(-?\d+)(\d{3})", '\g<1>,\g<2>', amount) @@ -643,7 +666,8 @@ else: return commaify(new) -def roundify(amount, sfs = 2): + +def roundify(amount, sfs=2): """ Take a number in string form and truncate to 'sfs' significant figures. """ @@ -652,6 +676,7 @@ else: return amount[0:sfs] + '0'*(len(amount) - sfs) + def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ): """ Returns a unicode string or None @@ -691,6 +716,7 @@ def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) + def string_to_object( s ): return pickle.loads( binascii.unhexlify( s ) ) @@ -743,19 +769,23 @@ return False return True + def get_ucsc_by_build(build): sites = [] for site in ucsc_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + + def get_gbrowse_sites_by_build(build): sites = [] for site in gbrowse_build_sites: if build in site['builds']: - sites.append((site['name'],site['url'])) + sites.append((site['name'], site['url'])) return sites + def read_dbnames(filename): """ Read build names from file """ class DBNames( list ): @@ -764,30 +794,34 @@ db_names = DBNames() try: ucsc_builds = {} - man_builds = [] #assume these are integers + man_builds = [] # assume these are integers name_to_db_base = {} for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") - #Special case of unspecified build is at top of list + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + # Special case of unspecified build is at top of list if fields[0] == "?": - db_names.insert(0,(fields[0],fields[1])) + db_names.insert(0, (fields[0], fields[1])) continue - try: #manual build (i.e. microbes) + try: # manual build (i.e. microbes) int(fields[0]) man_builds.append((fields[1], fields[0])) - except: #UCSC build + except: # UCSC build db_base = fields[0].rstrip('0123456789') if db_base not in ucsc_builds: ucsc_builds[db_base] = [] name_to_db_base[fields[1]] = db_base - #we want to sort within a species numerically by revision number + # we want to sort within a species numerically by revision number build_rev = re.compile(r'\d+$') - try: build_rev = int(build_rev.findall(fields[0])[0]) - except: build_rev = 0 - ucsc_builds[db_base].append((build_rev, fields[0],fields[1])) - except: continue + try: + build_rev = int(build_rev.findall(fields[0])[0]) + except: + build_rev = 0 + ucsc_builds[db_base].append((build_rev, fields[0], fields[1])) + except: + continue sort_names = name_to_db_base.keys() sort_names.sort() for name in sort_names: @@ -796,16 +830,18 @@ ucsc_builds[db_base].reverse() ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] db_names = DBNames( db_names + ucsc_builds[db_base] ) - if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) + if len( db_names ) > 1 and len( man_builds ) > 0: + db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) man_builds.sort() - man_builds = [(build, name) for name, build in man_builds] + man_builds = [(build, name) for name, build in man_builds] db_names = DBNames( db_names + man_builds ) except Exception, e: print "ERROR: Unable to read builds file:", e - if len(db_names)<1: + if len(db_names) < 1: db_names = DBNames( [( db_names.default_value, db_names.default_name )] ) return db_names + def read_ensembl( filename, ucsc ): """ Read Ensembl build names from file """ ucsc_builds = [] @@ -814,47 +850,55 @@ ensembl_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") - if fields[0] in ucsc_builds: continue + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") + if fields[0] in ucsc_builds: + continue ensembl_builds.append( dict( dbkey=fields[0], release=fields[1], name=fields[2].replace( '_', ' ' ) ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ensembl_builds + def read_ncbi( filename ): """ Read NCBI build names from file """ ncbi_builds = list() try: for line in open( filename ): - if line[0:1] in [ '#', '\t' ]: continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] in [ '#', '\t' ]: + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") ncbi_builds.append( dict( dbkey=fields[0], name=fields[1] ) ) except Exception, e: print "ERROR: Unable to read builds file:", e return ncbi_builds + def read_build_sites( filename, check_builds=True ): """ read db names to ucsc mappings from file, this file should probably be merged with the one above """ build_sites = [] try: for line in open(filename): try: - if line[0:1] == "#": continue - fields = line.replace("\r","").replace("\n","").split("\t") + if line[0:1] == "#": + continue + fields = line.replace("\r", "").replace("\n", "").split("\t") site_name = fields[0] site = fields[1] if check_builds: site_builds = fields[2].split(",") - site_dict = {'name':site_name, 'url':site, 'builds':site_builds} + site_dict = {'name': site_name, 'url': site, 'builds': site_builds} else: - site_dict = {'name':site_name, 'url':site} + site_dict = {'name': site_name, 'url': site} build_sites.append( site_dict ) - except: continue + except: + continue except: - print "ERROR: Unable to read builds for site file %s" %filename + print "ERROR: Unable to read builds for site file %s" % filename return build_sites + def relativize_symlinks( path, start=None, followlinks=False): for root, dirs, files in os.walk( path, followlinks=followlinks ): rel_start = None @@ -871,23 +915,26 @@ os.remove( symlink_file_name ) os.symlink( rel_path, symlink_file_name ) + def stringify_dictionary_keys( in_dict ): - #returns a new dictionary - #changes unicode keys into strings, only works on top level (does not recurse) - #unicode keys are not valid for expansion into keyword arguments on method calls + # returns a new dictionary + # changes unicode keys into strings, only works on top level (does not recurse) + # unicode keys are not valid for expansion into keyword arguments on method calls out_dict = {} for key, value in in_dict.iteritems(): out_dict[ str( key ) ] = value return out_dict + def recursively_stringify_dictionary_keys( d ): if isinstance(d, dict): - return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) + return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k, v in d.iteritems()]) elif isinstance(d, list): return [recursively_stringify_dictionary_keys(x) for x in d] else: return d + def mkstemp_ln( src, prefix='mkstemp_ln_' ): """ From tempfile._mkstemp_inner, generate a hard link in the same dir with a @@ -904,9 +951,10 @@ return (os.path.abspath(file)) except OSError, e: if e.errno == errno.EEXIST: - continue # try again + continue # try again raise - raise IOError, (errno.EEXIST, "No usable temporary file name found") + raise IOError(errno.EEXIST, "No usable temporary file name found") + def umask_fix_perms( path, umask, unmasked_perms, gid=None ): """ @@ -923,7 +971,7 @@ try: os.chmod( path, perms ) except Exception, e: - log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), \ + log.warning( 'Unable to honor umask (%s) for %s, tried to set: %s but mode remains %s, error was: %s' % ( oct( umask ), path, oct( perms ), oct( stat.S_IMODE( st.st_mode ) ), @@ -939,11 +987,12 @@ except: desired_group = gid current_group = st.st_gid - log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, \ + log.warning( 'Unable to honor primary group (%s) for %s, group remains %s, error was: %s' % ( desired_group, path, current_group, e ) ) + def docstring_trim(docstring): """Trimming python doc strings. Taken from: http://www.python.org/dev/peps/pep-0257/""" if not docstring: @@ -970,6 +1019,7 @@ # Return a single string: return '\n'.join(trimmed) + def nice_size(size): """ Returns a readably formatted string with the size @@ -989,14 +1039,15 @@ except: return '??? bytes' for ind, word in enumerate(words): - step = 1024 ** (ind + 1) + step = 1024 ** (ind + 1) if step > size: size = size / float(1024 ** ind) - if word == 'bytes': # No decimals for bytes + if word == 'bytes': # No decimals for bytes return "%d bytes" % size return "%.1f %s" % (size, word) return '??? bytes' + def size_to_bytes( size ): """ Returns a number of bytes if given a reasonably formatted string with the size @@ -1023,6 +1074,7 @@ elif multiple.startswith( 'b' ): return int( size ) + def send_mail( frm, to, subject, body, config ): """ Sends an email. @@ -1072,6 +1124,7 @@ s.sendmail( frm, to, msg.as_string() ) s.quit() + def force_symlink( source, link_name ): try: os.symlink( source, link_name ) @@ -1082,12 +1135,13 @@ else: raise e + def move_merge( source, target ): - #when using shutil and moving a directory, if the target exists, - #then the directory is placed inside of it - #if the target doesn't exist, then the target is made into the directory - #this makes it so that the target is always the target, and if it exists, - #the source contents are moved into the target + # when using shutil and moving a directory, if the target exists, + # then the directory is placed inside of it + # if the target doesn't exist, then the target is made into the directory + # this makes it so that the target is always the target, and if it exists, + # the source contents are moved into the target if os.path.isdir( source ) and os.path.exists( target ) and os.path.isdir( target ): for name in os.listdir( source ): move_merge( os.path.join( source, name ), os.path.join( target, name ) ) @@ -1103,7 +1157,7 @@ rv |= ord(x) ^ ord(y) return rv == 0 -galaxy_root_path = os.path.join(__path__[0], "..","..","..") +galaxy_root_path = os.path.join(__path__[0], "..", "..", "..") # The dbnames list is used in edit attributes and the upload tool dbnames = read_dbnames( os.path.join( galaxy_root_path, "tool-data", "shared", "ucsc", "builds.txt" ) ) @@ -1114,6 +1168,7 @@ gbrowse_build_sites = read_build_sites( os.path.join( galaxy_root_path, "tool-data", "shared", "gbrowse", "gbrowse_build_sites.txt" ) ) dlnames = dict(ucsc=ucsc_names, ensembl=ensembl_names, ncbi=ncbi_names) + def galaxy_directory(): return os.path.abspath(galaxy_root_path) https://bitbucket.org/galaxy/galaxy-central/commits/f03f9c5a5efc/ Changeset: f03f9c5a5efc User: dannon Date: 2014-07-24 23:53:37 Summary: Prevent redefinition of build_rev in list comprehension -- doesn't get used anyway Affected #: 1 file diff -r dd39f52ba6fc0cea9328d1437842904ac1d020c9 -r f03f9c5a5efc7fecb4ca131f66c441d55cc5d353 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -828,7 +828,7 @@ db_base = name_to_db_base[name] ucsc_builds[db_base].sort() ucsc_builds[db_base].reverse() - ucsc_builds[db_base] = [(build, name) for build_rev, build, name in ucsc_builds[db_base]] + ucsc_builds[db_base] = [(build, name) for _, build, name in ucsc_builds[db_base]] db_names = DBNames( db_names + ucsc_builds[db_base] ) if len( db_names ) > 1 and len( man_builds ) > 0: db_names.append( ( db_names.default_value, '----- Additional Species Are Below -----' ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for parsing tool <input> attributes when it contains no parameters.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/832bbb3fa9fb/ Changeset: 832bbb3fa9fb User: dan Date: 2014-07-24 22:46:24 Summary: Fix for parsing tool <input> attributes when it contains no parameters. Affected #: 1 file diff -r 6f7d70f72da237e0d8d37a20b14a4eab1828803d -r 832bbb3fa9fb62d53f2be23cf880755fa2001f46 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1391,7 +1391,7 @@ # Load parameters (optional) input_elem = root.find("inputs") enctypes = set() - if input_elem: + if input_elem is not None: # Handle properties of the input form self.check_values = string_as_bool( input_elem.get("check_values", self.check_values ) ) self.nginx_upload = string_as_bool( input_elem.get( "nginx_upload", self.nginx_upload ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Bugfix in test tool demoing two collection params.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/6f7d70f72da2/ Changeset: 6f7d70f72da2 User: jmchilton Date: 2014-07-24 22:30:51 Summary: Bugfix in test tool demoing two collection params. Bit problematic functional tests passed despite this bug. Affected #: 1 file diff -r b2700f28372549a17f5ca03af60e86964665c3ff -r 6f7d70f72da237e0d8d37a20b14a4eab1828803d test/functional/tools/collection_two_paired.xml --- a/test/functional/tools/collection_two_paired.xml +++ b/test/functional/tools/collection_two_paired.xml @@ -20,8 +20,8 @@ <param name="f2" type="data_collection" collection_type="paired" /></when><when value="list"> - <param name="f1" type="data_collection" collection_type="paired" /> - <param name="f2" type="data_collection" collection_type="paired" /> + <param name="f1" type="data_collection" collection_type="list" /> + <param name="f2" type="data_collection" collection_type="list" /></when></conditional></inputs> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add functional tool tests exercising multiple collection parameters at once.
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b2700f283725/ Changeset: b2700f283725 User: jmchilton Date: 2014-07-24 22:06:04 Summary: Add functional tool tests exercising multiple collection parameters at once. Affected #: 2 files diff -r 2be89285509068e862d27d41c3ba02c5a59e5b00 -r b2700f28372549a17f5ca03af60e86964665c3ff test/functional/tools/collection_two_paired.xml --- /dev/null +++ b/test/functional/tools/collection_two_paired.xml @@ -0,0 +1,69 @@ +<tool id="collection_two_paired" name="collection_two_paired" version="0.1.0"> + <command> + #if $kind.collection_type == "paired" + cat $kind.f1.forward $kind.f1['reverse'] >> $out1; + cat $kind.f2.forward $kind.f2['reverse'] >> $out1; + #else + #for $i, $_ in enumerate($kind.f1): + cat $kind.f1[$i] $kind.f2[$i] >> $out1; + #end for + #end if + </command> + <inputs> + <conditional name="kind"> + <param type="select" name="collection_type"> + <option value="paired">Paired Datasets</option> + <option value="list">List of Datasets</option> + </param> + <when value="paired"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + <when value="list"> + <param name="f1" type="data_collection" collection_type="paired" /> + <param name="f2" type="data_collection" collection_type="paired" /> + </when> + </conditional> + </inputs> + <outputs> + <data format="txt" name="out1" /> + </outputs> + <tests> + <test> + <conditional name="kind"> + <param name="collection_type" value="paired" /> + <param name="f1"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="paired"> + <element name="forward" value="simple_line.txt" /> + <element name="reverse" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + <test> + <conditional name="kind"> + <param name="collection_type" value="list" /> + <param name="f1"> + <collection type="list"> + <element name="l11" value="simple_line.txt" /> + <element name="l12" value="simple_line.txt" /> + </collection> + </param> + <param name="f2"> + <collection type="list"> + <element name="l21" value="simple_line_alternative.txt" /> + <element name="l22" value="simple_line_alternative.txt" /> + </collection> + </param> + </conditional> + <output name="out1" file="simple_lines_interleaved.txt"/> + </test> + </tests> +</tool> diff -r 2be89285509068e862d27d41c3ba02c5a59e5b00 -r b2700f28372549a17f5ca03af60e86964665c3ff test/functional/tools/samples_tool_conf.xml --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -21,4 +21,5 @@ <tool file="collection_paired_test.xml" /><tool file="collection_nested_test.xml" /><tool file="collection_mixed_param.xml" /> + <tool file="collection_two_paired.xml" /></toolbox> \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 24 Jul '14

24 Jul '14

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/20dcff34ee3b/ Changeset: 20dcff34ee3b User: jmchilton Date: 2014-07-24 21:30:11 Summary: More collection-y state update tests. Affected #: 1 file diff -r 9d8967752787556b3d814e086fe2bd9eb1992170 -r 20dcff34ee3bb598969c51a7c5b53e9dd679c111 test/unit/tools/test_execution.py --- a/test/unit/tools/test_execution.py +++ b/test/unit/tools/test_execution.py @@ -16,13 +16,12 @@ eggs.require( "Paste" ) from paste import httpexceptions -# Tool with a repeat parameter, to test state update. -REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"> +BASE_REPEAT_TOOL_CONTENTS = '''<tool id="test_tool" name="Test Tool"><command>echo "$param1" #for $r in $repeat# "$r.param2" #end for# < $out1</command><inputs><param type="text" name="param1" value="" /><repeat name="repeat1" label="Repeat 1"> - <param type="text" name="param2" value="" /> + %s </repeat></inputs><outputs> @@ -31,6 +30,10 @@ </tool> ''' +# Tool with a repeat parameter, to test state update. +REPEAT_TOOL_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="text" name="param2" value="" />''' +REPEAT_COLLECTION_PARAM_CONTENTS = BASE_REPEAT_TOOL_CONTENTS % '''<param type="data_collection" name="param2" collection_type="paired" />''' + class ToolExecutionTestCase( TestCase, tools_support.UsesApp, tools_support.UsesTools ): @@ -287,13 +290,48 @@ } ) self.__assert_exeuted( template, template_vars ) - def __history_dataset_collection_for( self, hdas, id=1234 ): - collection = galaxy.model.DatasetCollection() + def test_subcollection_multirun_with_state_updates( self ): + self._init_tool( REPEAT_COLLECTION_PARAM_CONTENTS ) + hda1, hda2 = self.__add_dataset( 1 ), self.__add_dataset( 2 ) + collection = self.__history_dataset_collection_for( [ hda1, hda2 ], collection_type="list:paired" ) + collection_id = self.app.security.encode_id( collection.id ) + self.app.dataset_collections_service = Bunch( + match_collections=lambda collections: None + ) + template, template_vars = self.__handle_with_incoming( + repeat1_add="dummy", + ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert len( state.inputs[ "repeat1" ] ) == 1 + template, template_vars = self.__handle_with_incoming( state, **{ + "repeat1_0|param2|__collection_multirun__": "%s|paired" % collection_id, + "repeat1_add": "dummy", + } ) + state = self.__assert_rerenders_tool_without_errors( template, template_vars ) + assert state.inputs[ "repeat1" ][ 0 ][ "param2|__collection_multirun__" ] == "%s|paired" % collection_id + + def __history_dataset_collection_for( self, hdas, collection_type="list", id=1234 ): + collection = galaxy.model.DatasetCollection( + collection_type=collection_type, + ) to_element = lambda hda: galaxy.model.DatasetCollectionElement( collection=collection, element=hda, ) - collection.datasets = map(to_element, hdas) + elements = map(to_element, hdas) + if collection_type == "list:paired": + paired_collection = galaxy.model.DatasetCollection( + collection_type="paired", + ) + paired_collection.elements = elements + list_dce = galaxy.model.DatasetCollectionElement( + collection=collection, + element=paired_collection, + ) + elements = [ list_dce ] + + collection.elements = elements + history_dataset_collection_association = galaxy.model.HistoryDatasetCollectionAssociation( id=id, collection=collection, @@ -349,13 +387,13 @@ self.history.datasets.append( hda ) return hda - def __add_collection_dataset( self, id, *hdas ): + def __add_collection_dataset( self, id, collection_type="paired", *hdas ): hdca = galaxy.model.HistoryDatasetCollectionAssociation() hdca.id = id collection = galaxy.model.DatasetCollection() hdca.collection = collection collection.elements = [ galaxy.model.DatasetCollectionElement(element=self.__add_dataset( 1 )) ] - + collection.type = collection_type self.trans.sa_session.model_objects[ galaxy.model.HistoryDatasetCollectionAssociation ][ id ] = hdca self.history.dataset_collections.append( hdca ) return hdca https://bitbucket.org/galaxy/galaxy-central/commits/9c2cbf6c7f3d/ Changeset: 9c2cbf6c7f3d User: jmchilton Date: 2014-07-24 21:30:11 Summary: Fix multirun and collection multirun state updates in tool param GUI. 4139740 fixed backend and had tests so I guess I didn't even manually test, turns out tool parameter GUI generation needed to fixed for these changes though. Affected #: 1 file diff -r 20dcff34ee3bb598969c51a7c5b53e9dd679c111 -r 9c2cbf6c7f3db7cb59ebd53b9c2829404077993a lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1712,10 +1712,15 @@ if self.__display_multirun_option(): # Select multiple datasets, run multiple jobs. multirun_key = "%s|__multirun__" % self.name + collection_multirun_key = "%s|__collection_multirun__" % self.name if multirun_key in (other_values or {}): multirun_value = listify( other_values[ multirun_key ] ) if multirun_value and len( multirun_value ) > 1: default_field = "select_multiple" + elif collection_multirun_key in (other_values or {}): + multirun_value = listify( other_values[ collection_multirun_key ] ) + if multirun_value: + default_field = "select_collection" else: multirun_value = value multi_dataset_matcher = DatasetMatcher( trans, self, multirun_value, other_values ) @@ -2014,9 +2019,17 @@ default_field = "select_single_collection" fields = odict() + collection_multirun_key = "%s|__collection_multirun__" % self.name + if collection_multirun_key in (other_values or {}): + multirun_value = other_values[ collection_multirun_key ] + if multirun_value: + default_field = "select_map_over_collections" + else: + multirun_value = value + history = self._get_history( trans ) fields[ "select_single_collection" ] = self._get_single_collection_field( trans=trans, history=history, value=value, other_values=other_values ) - fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=value, other_values=other_values ) + fields[ "select_map_over_collections" ] = self._get_select_dataset_collection_field( trans=trans, history=history, value=multirun_value, other_values=other_values ) return self._switch_fields( fields, default_field=default_field ) https://bitbucket.org/galaxy/galaxy-central/commits/2be892855090/ Changeset: 2be892855090 User: jmchilton Date: 2014-07-24 21:30:11 Summary: Small UI fix for data param multirun options. Index-based logic wasn't updated after made it so certain options wouldn't be visible if they were not valid for a given history. Affected #: 1 file diff -r 9c2cbf6c7f3db7cb59ebd53b9c2829404077993a -r 2be89285509068e862d27d41c3ba02c5a59e5b00 static/scripts/galaxy.tools.js --- a/static/scripts/galaxy.tools.js +++ b/static/scripts/galaxy.tools.js @@ -86,7 +86,7 @@ }).attr( 'title', selectionType['select_by'] - ); + ).data( "index", iIndex ); view.formRow().find( "label" ).append( button ); } }); @@ -114,11 +114,13 @@ } else { $("div#remap-row").css("display", "none"); } - this.formRow().find( "i" ).each(function(index, iElement) { + this.formRow().find( "i" ).each(function(_, iElement) { + var $iElement = $(iElement); + var index = $iElement.data("index"); if(index == enableIndex) { - $(iElement).css('color', 'black'); + $iElement.css('color', 'black'); } else { - $(iElement).css('color', 'Gray'); + $iElement.css('color', 'Gray'); } }); var $select = this.$( "select" ); Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0