January 2015 - galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: dannon: Merge from stable.
by commits-noreply＠bitbucket.org 27 Jan '15

27 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/21e5bf79d769/ Changeset: 21e5bf79d769 User: dannon Date: 2015-01-27 20:34:24+00:00 Summary: Merge from stable. Affected #: 1 file diff -r 4e6216506b6f480742d94b0ec0b6206c0e8ebc3d -r 21e5bf79d7690bc0ad0b3a4cca684327809caf60 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -1351,31 +1351,46 @@ id = galaxy.util.listify( id ) name = galaxy.util.listify( name ) histories = [] - cur_names = [] + for history_id in id: history = self.history_manager.get_owned( trans, self.decode_id( history_id ), trans.user ) if history and history.user_id == user.id: histories.append( history ) - cur_names.append( history.get_display_name() ) if not name or len( histories ) != len( name ): return trans.fill_template( "/history/rename.mako", histories=histories ) - change_msg = "" - for i in range(len(histories)): - if histories[i].user_id == user.id: - if name[i] == histories[i].get_display_name(): - change_msg = change_msg + "<p>History: "+cur_names[i]+" is already named: "+name[i]+"</p>" - elif name[i] not in [None,'',' ']: - name[i] = escape(name[i]) - histories[i].name = sanitize_html( name[i] ) - trans.sa_session.add( histories[i] ) - trans.sa_session.flush() - change_msg = change_msg + "<p>History: "+cur_names[i]+" renamed to: "+name[i]+"</p>" - trans.log_event( "History renamed: id: %s, renamed to: '%s'" % (str(histories[i].id), name[i] ) ) - else: - change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>" - else: - change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>" - return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) + + change_msgs = [] + for i in range( len( histories ) ): + cur_name = histories[i].get_display_name() + new_name = name[i] + + # skip if name is empty + if not isinstance( new_name, basestring ) or not new_name.strip(): + change_msgs.append( "You must specify a valid name for History: " + cur_name ) + continue + + # skip if not the owner + #??: isn't this already handled in get_history/if statement above? + if histories[i].user_id != user.id: + change_msgs.append( "History: " + cur_name + " does not appear to belong to you." ) + continue + + # skip if it wouldn't be a change + if new_name == cur_name: + change_msgs.append( "History: " + cur_name + " is already named: " + new_name ) + continue + + # escape, sanitize, set, and log the change + new_name = escape( new_name ) + histories[i].name = sanitize_html( new_name ) + trans.sa_session.add( histories[i] ) + trans.sa_session.flush() + + trans.log_event( "History renamed: id: %s, renamed to: '%s'" % ( str( histories[i].id ), new_name ) ) + change_msgs.append( "History: " + cur_name + " renamed to: " + new_name ) + + change_msg = '<br />'.join( change_msgs ) + return trans.show_message( change_msg, refresh_frames=['history'] ) @web.expose @web.require_login( "copy shared Galaxy history" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: Remove paragraph tags from history/rename feedback messages
by commits-noreply＠bitbucket.org 27 Jan '15

27 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1b96d3a4ff28/ Changeset: 1b96d3a4ff28 Branch: stable User: carlfeberhard Date: 2015-01-27 19:32:16+00:00 Summary: Remove paragraph tags from history/rename feedback messages Affected #: 1 file diff -r 11e14ac45c7ece4d6484f37be1123d7796cac599 -r 1b96d3a4ff28da2bd547b65ad86c5ad5a9bbfb90 lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -1371,31 +1371,46 @@ id = galaxy.util.listify( id ) name = galaxy.util.listify( name ) histories = [] - cur_names = [] + for history_id in id: history = self.get_history( trans, history_id ) if history and history.user_id == user.id: histories.append( history ) - cur_names.append( history.get_display_name() ) if not name or len( histories ) != len( name ): return trans.fill_template( "/history/rename.mako", histories=histories ) - change_msg = "" - for i in range(len(histories)): - if histories[i].user_id == user.id: - if name[i] == histories[i].get_display_name(): - change_msg = change_msg + "<p>History: "+cur_names[i]+" is already named: "+name[i]+"</p>" - elif name[i] not in [None,'',' ']: - name[i] = escape(name[i]) - histories[i].name = sanitize_html( name[i] ) - trans.sa_session.add( histories[i] ) - trans.sa_session.flush() - change_msg = change_msg + "<p>History: "+cur_names[i]+" renamed to: "+name[i]+"</p>" - trans.log_event( "History renamed: id: %s, renamed to: '%s'" % (str(histories[i].id), name[i] ) ) - else: - change_msg = change_msg + "<p>You must specify a valid name for History: "+cur_names[i]+"</p>" - else: - change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>" - return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) + + change_msgs = [] + for i in range( len( histories ) ): + cur_name = histories[i].get_display_name() + new_name = name[i] + + # skip if name is empty + if not isinstance( new_name, basestring ) or not new_name.strip(): + change_msgs.append( "You must specify a valid name for History: " + cur_name ) + continue + + # skip if not the owner + #??: isn't this already handled in get_history/if statement above? + if histories[i].user_id != user.id: + change_msgs.append( "History: " + cur_name + " does not appear to belong to you." ) + continue + + # skip if it wouldn't be a change + if new_name == cur_name: + change_msgs.append( "History: " + cur_name + " is already named: " + new_name ) + continue + + # escape, sanitize, set, and log the change + new_name = escape( new_name ) + histories[i].name = sanitize_html( new_name ) + trans.sa_session.add( histories[i] ) + trans.sa_session.flush() + + trans.log_event( "History renamed: id: %s, renamed to: '%s'" % ( str( histories[i].id ), new_name ) ) + change_msgs.append( "History: " + cur_name + " renamed to: " + new_name ) + + change_msg = '<br />'.join( change_msgs ) + return trans.show_message( change_msg, refresh_frames=['history'] ) @web.expose @web.require_login( "copy shared Galaxy history" ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Fix for re-displaying user registration form when registering as an email of an admin_user that is already declared in galaxy.ini (cntrller r dmb). NB: Automatic refresh of masthead after success did not seem to be occuring.
by commits-noreply＠bitbucket.org 27 Jan '15

27 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/4e6216506b6f/ Changeset: 4e6216506b6f User: dan Date: 2015-01-27 19:52:55+00:00 Summary: Fix for re-displaying user registration form when registering as an email of an admin_user that is already declared in galaxy.ini (cntrller r dmb). NB: Automatic refresh of masthead after success did not seem to be occuring. Affected #: 1 file diff -r df3b9b974104f827e8e50e7996a396ffdf2bc5bd -r 4e6216506b6f480742d94b0ec0b6206c0e8ebc3d templates/user/register.mako --- a/templates/user/register.mako +++ b/templates/user/register.mako @@ -17,7 +17,7 @@ ## An admin user may be creating a new user account, in which case we want to display the registration form. ## But if the current user is not an admin user, then don't display the registration form. -%if trans.user_is_admin() or not trans.user: +%if ( cntrller=='admin' and trans.user_is_admin() ) or not trans.user: ${render_registration_form()} %if trans.app.config.get( 'terms_url', None ) is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: jmchilton: Add comments to config/galaxy.ini.sample about recent workflow changes.
by commits-noreply＠bitbucket.org 27 Jan '15

27 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/df3b9b974104/ Changeset: df3b9b974104 User: jmchilton Date: 2015-01-27 18:48:01+00:00 Summary: Add comments to config/galaxy.ini.sample about recent workflow changes. Thanks for Philip Mabon for letting me know there were not included along with the original changes. Affected #: 1 file diff -r 5f59c9e4d36d796b40a4817c38b66f5def94dc2d -r df3b9b974104f827e8e50e7996a396ffdf2bc5bd config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -764,6 +764,25 @@ # Enable Galaxy to communicate directly with a sequencer #enable_sequencer_communication = False + +# Enable beta workflow modules that should not yet be considered part of Galaxy's +# stable API. +#enable_beta_workflow_modules = False + +# Force usage of Galaxy's beta workflow scheduler under certain circumstances - +# this workflow scheduling forces Galaxy to schedule workflows in the background +# so initial submission of the workflows is signficantly sped up. This does +# however force the user to refresh their history manually to see newly scheduled +# steps (for "normal" workflows - steps are still scheduled far in advance of +# them being queued and scheduling here doesn't refer to actual cluster job +# scheduling). +# Workflows containing more than the specified number of steps will always use +# the Galaxy's beta workflow scheduling. +#force_beta_workflow_scheduled_min_steps=250 +# Switch to using Galaxy's beta workflow scheduling for all workflows involving +# ccollections. +#force_beta_workflow_scheduled_for_collections=False + # Enable authentication via OpenID. Allows users to log in to their Galaxy # account by authenticating with an OpenID provider. #enable_openid = False Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

[galaxyproject/usegalaxy-playbook] bc784c: Update test.
by GitHub 27 Jan '15

27 Jan '15

Branch: refs/heads/master Home: https://github.com/galaxyproject/usegalaxy-playbook Commit: bc784c1bfc309ec59471dfcf2f928da0733562e9 https://github.com/galaxyproject/usegalaxy-playbook/commit/bc784c1bfc309ec5… Author: Dannon <dannon.baker(a)gmail.com> Date: 2015-01-27 (Tue, 27 Jan 2015) Changed paths: M stage/group_vars/all.yml Log Message: ----------- Update test.

1 0

commit/galaxy-central: jmchilton: Test cases for select validation handling.
by commits-noreply＠bitbucket.org 27 Jan '15

27 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/5f59c9e4d36d/ Changeset: 5f59c9e4d36d User: jmchilton Date: 2015-01-27 17:14:56+00:00 Summary: Test cases for select validation handling. Normal selects seem to be prevented from execution with invalid parameter values, but not columns. Values are escaped properly so shell exploitation isn't the problem - but as a usability thing Galaxy should prevent execution and provide a warning message. Affected #: 1 file diff -r 1096077cabbd4046b7da9ce45cecf1151b7ebc75 -r 5f59c9e4d36d796b40a4817c38b66f5def94dc2d test/api/test_tools.py --- a/test/api/test_tools.py +++ b/test/api/test_tools.py @@ -201,6 +201,27 @@ response = self._run( "validation_default", history_id, inputs ) self._assert_status_code_is( response, 400 ) + @skip_without_tool( "multi_select" ) + def test_select_legal_values( self ): + history_id = self.dataset_populator.new_history() + inputs = { + 'select_ex': 'not_option', + } + response = self._run( "multi_select", history_id, inputs ) + self._assert_status_code_is( response, 400 ) + + @skip_without_tool( "column_param" ) + def test_column_legal_values( self ): + history_id = self.dataset_populator.new_history() + new_dataset1 = self.dataset_populator.new_dataset( history_id, content='#col1\tcol2' ) + inputs = { + 'input1': { "src": "hda", "id": new_dataset1["id"] }, + 'col': "' ; echo 'moo", + } + response = self._run( "column_param", history_id, inputs ) + # TODO: make this test pass... + self._assert_status_code_is( response, 400 ) + @skip_without_tool( "collection_paired_test" ) def test_collection_parameter( self ): history_id = self.dataset_populator.new_history() Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 26 Jan '15

26 Jan '15

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a439922a4956/ Changeset: a439922a4956 User: dannon Date: 2015-01-26 21:25:19+00:00 Summary: Allow cancelling of upload jobs. This won't cancel the actual upload, but it'll kill (or prevent) the processing job if it exists. This also allows one to clean up 'zombie' upload jobs that never leave 'upload' state via the admin panel. Affected #: 1 file diff -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 -r a439922a4956cdef237925540f4a70922ae8a74c templates/admin/jobs.mako --- a/templates/admin/jobs.mako +++ b/templates/admin/jobs.mako @@ -55,11 +55,7 @@ </tr> %for job in jobs: <td> - %if job.state == 'upload': -   - %else: - <input type="checkbox" name="stop" value="${job.id}"/> - %endif + <input type="checkbox" name="stop" value="${job.id}"/></td><td>${job.id}</td> %if job.history and job.history.user: https://bitbucket.org/galaxy/galaxy-central/commits/1096077cabbd/ Changeset: 1096077cabbd User: dannon Date: 2015-01-26 21:25:29+00:00 Summary: Merge. Affected #: 2 files diff -r a439922a4956cdef237925540f4a70922ae8a74c -r 1096077cabbd4046b7da9ce45cecf1151b7ebc75 config/tool_data_table_conf.xml.sample --- a/config/tool_data_table_conf.xml.sample +++ b/config/tool_data_table_conf.xml.sample @@ -60,4 +60,9 @@ <columns>value, dbkey, name, path</columns><file path="tool-data/mosaik_index.loc" /></table> +  + <table name="twobit" comment_char="#"> + <columns>value, path</columns> + <file path="tool-data/twobit.loc" /> + </table></tables> diff -r a439922a4956cdef237925540f4a70922ae8a74c -r 1096077cabbd4046b7da9ce45cecf1151b7ebc75 lib/galaxy/visualization/genomes.py --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -179,33 +179,59 @@ """ def __init__( self, app ): + self.app = app # Create list of genomes from app.genome_builds self.genomes = {} - for key, description in app.genome_builds.get_genome_build_names(): + # Store internal versions of data tables for twobit and __dbkey__ + self._table_versions = { 'twobit': None, '__dbkeys__': None } + self.reload_genomes() + + def reload_genomes( self ): + self.genomes = {} + # Store table versions for later + for table_name in self._table_versions.keys(): + table = self.app.tool_data_tables.get( table_name, None ) + if table is not None: + self._table_versions[ table_name ] = table._loaded_content_version + + twobit_table = self.app.tool_data_tables.get( 'twobit', None ) + twobit_fields = {} + if twobit_table is None: + # Add genome data (twobit files) to genomes, directly from twobit.loc + try: + for line in open( os.path.join( self.app.config.tool_data_path, "twobit.loc" ) ): + if line.startswith("#"): continue + val = line.split() + if len( val ) == 2: + key, path = val + twobit_fields[ key ] = path + except IOError, e: + # Thrown if twobit.loc does not exist. + log.exception( "Error reading twobit.loc: %s", e ) + for key, description in self.app.genome_builds.get_genome_build_names(): self.genomes[ key ] = Genome( key, description ) + # Add len files to genomes. + self.genomes[ key ].len_file = self.app.genome_builds.get_chrom_info( key )[0] + if self.genomes[ key ].len_file: + if not os.path.exists( self.genomes[ key ].len_file ): + self.genomes[ key ].len_file = None + # Add genome data (twobit files) to genomes. + if twobit_table is not None: + self.genomes[ key ].twobit_file = twobit_table.get_entry( 'value', key, 'path', default=None ) + elif key in twobit_fields: + self.genomes[ key ].twobit_file = twobit_fields[ key ] + - # Add len files to genomes. - len_files = glob.glob( os.path.join( app.config.len_file_path, "*.len" ) ) - for f in len_files: - key = os.path.split( f )[1].split( ".len" )[0] - if key in self.genomes: - self.genomes[ key ].len_file = f - - # Add genome data (twobit files) to genomes. - try: - for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ): - if line.startswith("#"): continue - val = line.split() - if len( val ) == 2: - key, path = val - if key in self.genomes: - self.genomes[ key ].twobit_file = path - except IOError, e: - # Thrown if twobit.loc does not exist. - log.exception( str( e ) ) + def check_and_reload( self ): + # Check if tables have been modified, if so reload + for table_name, table_version in self._table_versions.iteritems(): + table = self.app.tool_data_tables.get( table_name, None ) + if table is not None and not table.is_current_version( table_version ): + return self.reload_genomes() def get_build( self, dbkey ): """ Returns build for the given key. """ + self.check_and_reload() rval = None if dbkey in self.genomes: rval = self.genomes[ dbkey ] @@ -214,6 +240,7 @@ def get_dbkeys( self, trans, chrom_info=False, **kwd ): """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ + self.check_and_reload() dbkeys = [] # Add user's custom keys to dbkeys. @@ -241,7 +268,7 @@ Returns a naturally sorted list of chroms/contigs for a given dbkey. Use either chrom or low to specify the starting chrom in the return list. """ - + self.check_and_reload() # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: @@ -303,6 +330,7 @@ Returns true if there is reference data for the specified dbkey. If dbkey is custom, dbkey_owner is needed to determine if there is reference data. """ + self.check_and_reload() # Look for key in built-in builds. if dbkey in self.genomes and self.genomes[ dbkey ].twobit_file: # There is built-in reference data. @@ -323,7 +351,7 @@ """ Return reference data for a build. """ - + self.check_and_reload() # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dan: Load genomes list from data tables for visualizations. Based off of Pull Request: https://bitbucket.org/galaxy/galaxy-central/pull-request/601/load-genomes-list-from-data-tables-for/diff
by commits-noreply＠bitbucket.org 26 Jan '15

26 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1ab223c3860e/ Changeset: 1ab223c3860e User: dan Date: 2015-01-26 20:11:06+00:00 Summary: Load genomes list from data tables for visualizations. Based off of Pull Request: https://bitbucket.org/galaxy/galaxy-central/pull-request/601/load-genomes-l… Affected #: 2 files diff -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 -r 1ab223c3860e533785527074f36e2d1e16a596f0 config/tool_data_table_conf.xml.sample --- a/config/tool_data_table_conf.xml.sample +++ b/config/tool_data_table_conf.xml.sample @@ -60,4 +60,9 @@ <columns>value, dbkey, name, path</columns><file path="tool-data/mosaik_index.loc" /></table> +  + <table name="twobit" comment_char="#"> + <columns>value, path</columns> + <file path="tool-data/twobit.loc" /> + </table></tables> diff -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 -r 1ab223c3860e533785527074f36e2d1e16a596f0 lib/galaxy/visualization/genomes.py --- a/lib/galaxy/visualization/genomes.py +++ b/lib/galaxy/visualization/genomes.py @@ -179,33 +179,59 @@ """ def __init__( self, app ): + self.app = app # Create list of genomes from app.genome_builds self.genomes = {} - for key, description in app.genome_builds.get_genome_build_names(): + # Store internal versions of data tables for twobit and __dbkey__ + self._table_versions = { 'twobit': None, '__dbkeys__': None } + self.reload_genomes() + + def reload_genomes( self ): + self.genomes = {} + # Store table versions for later + for table_name in self._table_versions.keys(): + table = self.app.tool_data_tables.get( table_name, None ) + if table is not None: + self._table_versions[ table_name ] = table._loaded_content_version + + twobit_table = self.app.tool_data_tables.get( 'twobit', None ) + twobit_fields = {} + if twobit_table is None: + # Add genome data (twobit files) to genomes, directly from twobit.loc + try: + for line in open( os.path.join( self.app.config.tool_data_path, "twobit.loc" ) ): + if line.startswith("#"): continue + val = line.split() + if len( val ) == 2: + key, path = val + twobit_fields[ key ] = path + except IOError, e: + # Thrown if twobit.loc does not exist. + log.exception( "Error reading twobit.loc: %s", e ) + for key, description in self.app.genome_builds.get_genome_build_names(): self.genomes[ key ] = Genome( key, description ) + # Add len files to genomes. + self.genomes[ key ].len_file = self.app.genome_builds.get_chrom_info( key )[0] + if self.genomes[ key ].len_file: + if not os.path.exists( self.genomes[ key ].len_file ): + self.genomes[ key ].len_file = None + # Add genome data (twobit files) to genomes. + if twobit_table is not None: + self.genomes[ key ].twobit_file = twobit_table.get_entry( 'value', key, 'path', default=None ) + elif key in twobit_fields: + self.genomes[ key ].twobit_file = twobit_fields[ key ] + - # Add len files to genomes. - len_files = glob.glob( os.path.join( app.config.len_file_path, "*.len" ) ) - for f in len_files: - key = os.path.split( f )[1].split( ".len" )[0] - if key in self.genomes: - self.genomes[ key ].len_file = f - - # Add genome data (twobit files) to genomes. - try: - for line in open( os.path.join( app.config.tool_data_path, "twobit.loc" ) ): - if line.startswith("#"): continue - val = line.split() - if len( val ) == 2: - key, path = val - if key in self.genomes: - self.genomes[ key ].twobit_file = path - except IOError, e: - # Thrown if twobit.loc does not exist. - log.exception( str( e ) ) + def check_and_reload( self ): + # Check if tables have been modified, if so reload + for table_name, table_version in self._table_versions.iteritems(): + table = self.app.tool_data_tables.get( table_name, None ) + if table is not None and not table.is_current_version( table_version ): + return self.reload_genomes() def get_build( self, dbkey ): """ Returns build for the given key. """ + self.check_and_reload() rval = None if dbkey in self.genomes: rval = self.genomes[ dbkey ] @@ -214,6 +240,7 @@ def get_dbkeys( self, trans, chrom_info=False, **kwd ): """ Returns all known dbkeys. If chrom_info is True, only dbkeys with chromosome lengths are returned. """ + self.check_and_reload() dbkeys = [] # Add user's custom keys to dbkeys. @@ -241,7 +268,7 @@ Returns a naturally sorted list of chroms/contigs for a given dbkey. Use either chrom or low to specify the starting chrom in the return list. """ - + self.check_and_reload() # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: @@ -303,6 +330,7 @@ Returns true if there is reference data for the specified dbkey. If dbkey is custom, dbkey_owner is needed to determine if there is reference data. """ + self.check_and_reload() # Look for key in built-in builds. if dbkey in self.genomes and self.genomes[ dbkey ].twobit_file: # There is built-in reference data. @@ -323,7 +351,7 @@ """ Return reference data for a build. """ - + self.check_and_reload() # If there is no dbkey owner, default to current user. dbkey_owner, dbkey = decode_dbkey( dbkey ) if dbkey_owner: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: martenson: initial version of TS search API and Index builder script
by commits-noreply＠bitbucket.org 26 Jan '15

26 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0733c87123cb/ Changeset: 0733c87123cb User: martenson Date: 2015-01-26 18:08:16+00:00 Summary: initial version of TS search API and Index builder script beware: always returns jsonp for now Affected #: 7 files diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 lib/galaxy/webapps/tool_shed/api/search.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/api/search.py @@ -0,0 +1,34 @@ +"""API for searching the toolshed repositories""" +from galaxy import exceptions +from galaxy import eggs +from galaxy.web import _future_expose_api as expose_api +# from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous +from galaxy.web import _future_expose_api_raw_anonymous as expose_api_raw_anonymous +from galaxy.web.base.controller import BaseAPIController +from galaxy.webapps.tool_shed.search.repo_search import RepoSearch +import json +import logging +log = logging.getLogger( __name__ ) + + +class SearchController ( BaseAPIController ): + + @expose_api_raw_anonymous + def search( self, trans, search_term, **kwd ): + """ + Perform a search over the Whoosh index. + The index has to be pre-created with build_ts_whoosh_index.sh. + TS config option toolshed_search_on has to be turned on and + toolshed_whoosh_index_dir has to be specified and existing. + """ + if not self.app.config.toolshed_search_on: + raise exceptions.ConfigDoesNotAllowException( 'Searching the TS through the API is turned off for this instance.' ) + if not self.app.config.toolshed_whoosh_index_dir: + raise exceptions.ConfigDoesNotAllowException( 'There is no directory for the search index specified. Please ontact the administrator.' ) + + repo_search = RepoSearch() + results = repo_search.search( trans, search_term ) + + response = '%s(%s);' % ( kwd.get('callback'), json.dumps(results) ) + log.debug(response) + return response diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -85,6 +85,11 @@ controller='authenticate', action='get_tool_shed_api_key', conditions=dict( method=[ "GET" ] ) ) + webapp.mapper.connect( 'repo_search', + '/api/search/', + controller='search', + action='search', + conditions=dict( method=[ "GET" ] ) ) webapp.mapper.resource( 'category', 'categories', controller='categories', diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 lib/galaxy/webapps/tool_shed/config.py --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -40,6 +40,9 @@ self.database_connection = kwargs.get( "database_connection", False ) self.database_engine_options = get_database_engine_options( kwargs ) self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) ) + # Whoosh search + self.toolshed_search_on = string_as_bool( kwargs.get( "toolshed_search_on", False ) ) + self.toolshed_whoosh_index_dir = kwargs.get( "toolshed_whoosh_index_dir", None ) # Analytics self.ga_code = kwargs.get( "ga_code", None ) # Where dataset files are stored diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 lib/galaxy/webapps/tool_shed/search/repo_search.py --- /dev/null +++ b/lib/galaxy/webapps/tool_shed/search/repo_search.py @@ -0,0 +1,98 @@ +"""Module for searching the toolshed repositories""" +from galaxy import exceptions +from galaxy import eggs +from galaxy.web.base.controller import BaseAPIController +from galaxy.webapps.tool_shed import model +from tool_shed.util.shed_util_common import generate_sharable_link_for_repository_in_tool_shed + + +import logging +log = logging.getLogger( __name__ ) + +# Whoosh is compatible with Python 2.5+ +# Try to import Whoosh and set flag to indicate whether +# the tool search is ready. +try: + eggs.require( "Whoosh" ) + import whoosh.index + from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT + from whoosh.scoring import BM25F + from whoosh.qparser import MultifieldParser + from whoosh.index import Index + search_ready = True + + schema = Schema( + id=STORED, + name=TEXT, + description=TEXT, + long_description=TEXT, + repo_type=TEXT, + homepage_url=TEXT, + remote_repository_url=TEXT, + repo_owner_username=TEXT ) +except ImportError, e: + search_ready = False + schema = None + +class RepoSearch ( object ): + + def search( self, trans, search_term, **kwd ): + """ + Perform the search on the given search_term + + :param search_term: unicode encoded string with the search term(s) + + :returns results: dictionary containing number of hits, + hits themselves and matched terms for each + """ + if search_ready: + toolshed_whoosh_index_dir = trans.app.config.toolshed_whoosh_index_dir + index_exists = whoosh.index.exists_in( toolshed_whoosh_index_dir ) + if index_exists: + index = whoosh.index.open_dir( toolshed_whoosh_index_dir ) + try: + # Some literature about BM25F: + # http://trec.nist.gov/pubs/trec13/papers/microsoft-cambridge.web.hard.pdf + # http://en.wikipedia.org/wiki/Okapi_BM25 + # Basically the higher number the bigger weight. + searcher = index.searcher( weighting=BM25F( field_B={ + 'name_B' : 0.9, + 'description_B' : 0.6, + 'long_description_B' : 0.5, + 'homepage_url_B' : 0.3, + 'remote_repository_url_B' : 0.2, + 'repo_owner_username' : 0.3, + 'repo_type_B' : 0.1 } ) ) + parser = MultifieldParser( [ + 'name', + 'description', + 'long_description', + 'repo_type', + 'homepage_url', + 'remote_repository_url', + 'repo_owner_username' ], schema=schema ) + + hits = searcher.search( parser.parse( '*' + search_term + '*' ), terms = True ) + results = {} + results[ 'length'] = len( hits ) + results[ 'hits' ] = [] + for hit in hits: + repo = trans.sa_session.query( model.Repository ).filter_by( id=hit[ 'id' ] ).one() + approved = 'no' + for review in repo.reviews: + if review.approved == 'yes': + approved = 'yes' + break + hit_dict = repo.to_dict( view='element', value_mapper={ 'id': trans.security.encode_id, 'user_id': trans.security.encode_id } ) + hit_dict[ 'url'] = generate_sharable_link_for_repository_in_tool_shed( repo ) + hit_dict[ 'last_updated' ] = repo.update_time.strftime( "%Y-%m-%d %I:%M %p" ) + hit_dict[ 'times_downloaded' ] = repo.times_downloaded + hit_dict[ 'approved' ] = approved + results[ 'hits' ].append( {'repository': hit_dict, 'matched_terms': hit.matched_terms() } ) + return results + finally: + searcher.close() + else: + raise exceptions.InternalServerError( 'The search index file is missing.' ) + else: + raise exceptions.InternalServerError( 'Could not initialize search.' ) diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 scripts/tool_shed/build_ts_whoosh_index.py --- /dev/null +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -0,0 +1,101 @@ +# !/usr/bin/env python +""" Build indeces for searching the TS """ +import sys, os, csv, urllib, urllib2, ConfigParser + +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path + +from galaxy import eggs + +# Whoosh is compatible with Python 2.5+ Try to import Whoosh and set flag to indicate whether search is enabled. +try: + eggs.require( "Whoosh" ) + import pkg_resources + pkg_resources.require( "SQLAlchemy >= 0.4" ) + + import whoosh.index + import galaxy.webapps.tool_shed.model.mapping + from whoosh.filedb.filestore import FileStorage + from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT + from whoosh.scoring import BM25F + from whoosh.qparser import MultifieldParser + from whoosh.index import Index + from galaxy.webapps.tool_shed import config, model + + whoosh_ready = True + schema = Schema( + id=STORED, + name=TEXT, + description=TEXT, + long_description=TEXT, + repo_type=TEXT, + homepage_url=TEXT, + remote_repository_url=TEXT, + repo_owner_username=TEXT ) + +except ImportError, e: + print 'import error' + whoosh_ready = False + schema = None + +def build_index( sa_session, toolshed_whoosh_index_dir ): + storage = FileStorage( toolshed_whoosh_index_dir ) + index = storage.create_index( schema ) + writer = index.writer() + def to_unicode( a_basestr ): + if type( a_basestr ) is str: + return unicode( a_basestr, 'utf-8' ) + else: + return a_basestr + + repos_indexed = 0 + for id, name, description, long_description, repo_type, homepage_url, remote_repository_url, repo_owner_username in get_repos( sa_session ): + writer.add_document( id=id, + name=to_unicode( name ), + description=to_unicode( description ), + long_description=to_unicode( long_description ), + repo_type=to_unicode( repo_type ), + homepage_url=to_unicode( homepage_url ), + remote_repository_url=to_unicode( remote_repository_url ), + repo_owner_username=to_unicode( repo_owner_username ) ) + repos_indexed += 1 + writer.commit() + print "Number of repos indexed: ", repos_indexed + +def get_repos( sa_session ): + for repo in sa_session.query( model.Repository ).filter_by( deleted=False ).filter_by( deprecated=False ).filter( model.Repository.type != 'tool_dependency_definition' ): + id = repo.id + name = repo.name + description = repo.description + long_description = repo.long_description + repo_type = repo.type + homepage_url = repo.homepage_url + remote_repository_url = repo.remote_repository_url + + repo_owner_username = "" + if repo.user_id is not None: + user = sa_session.query( model.User ).filter( model.User.id == repo.user_id ).one() + repo_owner_username = user.username + + yield id, name, description, long_description, repo_type, homepage_url, remote_repository_url, repo_owner_username + +def get_sa_session_and_needed_config_settings( ini_file ): + conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } ) + conf_parser.read( ini_file ) + kwds = dict() + for key, value in conf_parser.items( "app:main" ): + kwds[ key ] = value + config_settings = config.Configuration( **kwds ) + db_con = config_settings.database_connection + if not db_con: + db_con = "sqlite:///%s?isolation_level=IMMEDIATE" % config_settings.database + model = galaxy.webapps.tool_shed.model.mapping.init( config_settings.file_path, db_con, engine_options={}, create_tables=False ) + return model.context.current, config_settings + +if __name__ == "__main__": + if whoosh_ready: + ini_file = sys.argv[ 1 ] + sa_session, config_settings = get_sa_session_and_needed_config_settings( ini_file ) + toolshed_whoosh_index_dir = config_settings.get( 'toolshed_whoosh_index_dir', None ) + build_index( sa_session, toolshed_whoosh_index_dir ) diff -r 418633a366083e171f2a3583b591d81a7b333c41 -r 0733c87123cbc2f7009a125c5e57d630bb8296a6 scripts/tool_shed/build_ts_whoosh_index.sh --- /dev/null +++ b/scripts/tool_shed/build_ts_whoosh_index.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cd `dirname $0`/../.. +# Make sure your config is at config/tool_shed.ini +# and that you specified toolshed_whoosh_index_dir in it. +python scripts/tool_shed/build_ts_whoosh_index.py config/tool_shed.ini Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: One more path permission increase for admins under remote_user.
by commits-noreply＠bitbucket.org 26 Jan '15

26 Jan '15

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/418633a36608/ Changeset: 418633a36608 User: dannon Date: 2015-01-26 15:43:02+00:00 Summary: One more path permission increase for admins under remote_user. Affected #: 1 file diff -r 39f3f0586c48f81452e3a7a6af1f5e97c4ef8ed0 -r 418633a366083e171f2a3583b591d81a7b333c41 lib/galaxy/web/framework/middleware/remoteuser.py --- a/lib/galaxy/web/framework/middleware/remoteuser.py +++ b/lib/galaxy/web/framework/middleware/remoteuser.py @@ -116,6 +116,8 @@ pass # Admin users may be impersonating, allow logout. elif path_info.startswith( '/user/manage_user_info' ) and environ[ self.remote_user_header ] in self.admin_users: pass # Admin users need to be able to change user information + elif path_info.startswith( '/user/edit_info' ) and environ[ self.remote_user_header ] in self.admin_users: + pass # Admin users need to be able to change user information elif path_info.startswith( '/user/api_keys' ): pass # api keys can be managed when remote_user is in use elif path_info.startswith( '/user/edit_username' ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0