[galaxy-commits] commit/galaxy-central: greg: Refine the definition of changeset revisions that can be automatically installed into a local Galaxy instance.

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/changeset/0ffcc9170823/ changeset: 0ffcc9170823 user: greg date: 2012-07-05 17:55:10 summary: Refine the definition of changeset revisions that can be automatically installed into a local Galaxy instance. affected #: 7 files diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/util/shed_util.py --- a/lib/galaxy/util/shed_util.py +++ b/lib/galaxy/util/shed_util.py @@ -76,8 +76,9 @@ processed_invalid_tool_configs = [] processed_relative_workflow_paths = [] processed_datatypes = [] - for downloadable_revision in repository.downloadable_revisions: - metadata = downloadable_revision.metadata + # A repository's metadata_revisions are those that ignore the value of the repository_metadata.downloadable column. + for metadata_revision in repository.metadata_revisions: + metadata = metadata_revision.metadata if 'tools' in metadata: tool_dicts = metadata[ 'tools' ] for tool_dict in tool_dicts: @@ -1461,7 +1462,7 @@ """ # To set excluded_lower_bounds_changeset_revision, calling methods should do the following, where the value of changeset_revision # is a downloadable changeset_revision. - # excluded_lower_bounds_changeset_revision = get_previous_valid_changset_revision( repository, repo, changeset_revision ) + # excluded_lower_bounds_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) if excluded_lower_bounds_changeset_revision == INITIAL_CHANGELOG_HASH: appending_started = True else: diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/controllers/common.py --- a/lib/galaxy/webapps/community/controllers/common.py +++ b/lib/galaxy/webapps/community/controllers/common.py @@ -134,10 +134,7 @@ trans.sa_session.add( repository_metadata ) trans.sa_session.flush() def build_changeset_revision_select_field( trans, repository, selected_value=None, add_id_to_name=True ): - """ - Build a SelectField whose options are the changeset_revision - strings of all downloadable_revisions of the received repository. - """ + """Build a SelectField whose options are the changeset_rev strings of all downloadable revisions of the received repository.""" repo = hg.repository( get_configured_ui(), repository.repo_path ) options = [] changeset_tups = [] @@ -169,6 +166,10 @@ selected = selected_value and option_tup[1] == selected_value select_field.add_option( option_tup[0], option_tup[1], selected=selected ) return select_field +def changeset_is_downloadable( metadata_dict ): + # A RepositoryMetadata record will be created if metadata_dict includes only invalid stuff like 'invalid_tools', but in this case + # it won't be downloadable. + return 'datatypes' in metadata_dict or 'tools' in metadata_dict or 'workflows' in metadata_dict def changeset_is_malicious( trans, id, changeset_revision, **kwd ): """Check the malicious flag in repository metadata for a specified change set""" repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) @@ -374,15 +375,12 @@ def create_or_update_repository_metadata( trans, id, repository, changeset_revision, metadata_dict ): repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: - # Update RepositoryMetadata.metadata. repository_metadata.metadata = metadata_dict - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() else: - # Create a new repository_metadata table row. repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) - trans.sa_session.add( repository_metadata ) - trans.sa_session.flush() + repository_metadata.downloadable = changeset_is_downloadable( metadata_dict ) + trans.sa_session.add( repository_metadata ) + trans.sa_session.flush() def decode( value ): # Extract and verify hash a, b = value.split( ":" ) @@ -605,7 +603,7 @@ if parent_id is None: # The tool did not change through all of the changeset revisions. return old_id -def get_previous_valid_changset_revision( repository, repo, before_changeset_revision ): +def get_previous_downloadable_changset_revision( repository, repo, before_changeset_revision ): """ Return the downloadable changeset_revision in the repository changelog just prior to the changeset to which before_changeset_revision refers. If there isn't one, return the hash value of an empty repository changlog, INITIAL_CHANGELOG_HASH. @@ -1087,12 +1085,9 @@ # Update the last saved repository_metadata table row. repository_metadata.changeset_revision = changeset_revision repository_metadata.metadata = metadata_dict + repository_metadata.downloadable = changeset_is_downloadable( metadata_dict ) trans.sa_session.add( repository_metadata ) - try: - trans.sa_session.flush() - except TypeError, e: - message = "Unable to save metadata for this repository probably due to a tool config file that doesn't conform to the Cheetah template syntax." - status = 'error' + trans.sa_session.flush() else: # There are no tools in the repository, and we're setting metadata on the repository tip. repository_metadata = trans.model.RepositoryMetadata( repository.id, changeset_revision, metadata_dict ) @@ -1102,6 +1097,7 @@ # We're re-generating metadata for an old repository revision. repository_metadata = get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) repository_metadata.metadata = metadata_dict + repository_metadata.downloadable = changeset_is_downloadable( metadata_dict ) trans.sa_session.add( repository_metadata ) trans.sa_session.flush() elif updating_tip and len( repo ) == 1 and not invalid_files: diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/controllers/repository.py --- a/lib/galaxy/webapps/community/controllers/repository.py +++ b/lib/galaxy/webapps/community/controllers/repository.py @@ -75,10 +75,7 @@ def __init__( self, col_name ): grids.GridColumn.__init__( self, col_name ) def get_value( self, trans, grid, repository ): - """ - Display a SelectField whose options are the changeset_revision - strings of all downloadable_revisions of this repository. - """ + """Display a SelectField whose options are the changeset_revision strings of all downloadable_revisions of this repository.""" select_field = build_changeset_revision_select_field( trans, repository ) if len( select_field.options ) > 1: return select_field.get_html() @@ -245,22 +242,10 @@ filterable="standard" ) ) operations = [] def build_initial_query( self, trans, **kwd ): - # The clause_list approach is to filter out those repositories that include metadata, but only because they contain 'invalid_tools' in - # the metadata (i.e., they don't have valid tools, datatypes or workflows). Is there a better approach? - clause_list = [] - for repository_metadata in trans.sa_session.query( trans.model.RepositoryMetadata ) \ - .filter( trans.model.RepositoryMetadata.table.c.malicious == False ): - metadata = repository_metadata.metadata - if 'datatypes' in metadata or'tools' in metadata or 'workflows' in metadata: - clause_list.append( trans.model.RepositoryMetadata.table.c.id == repository_metadata.id ) - if clause_list: - return trans.sa_session.query( self.model_class ) \ - .join( model.RepositoryMetadata.table ) \ - .join( model.User.table ) \ - .filter( or_( *clause_list ) ) return trans.sa_session.query( self.model_class ) \ .join( model.RepositoryMetadata.table ) \ - .join( model.User.table ) + .join( model.User.table ) \ + .filter( model.RepositoryMetadata.table.c.downloadable == True ) class MatchedRepositoryListGrid( grids.Grid ): class NameColumn( grids.TextColumn ): @@ -400,7 +385,8 @@ for repository in trans.sa_session.query( trans.model.Repository ) \ .filter( trans.model.Repository.table.c.deleted == False ) \ .order_by( trans.model.Repository.table.c.name ): - for downloadable_revision in repository.downloadable_revisions: + # A repository's metadata_revisions are those that ignore the value of the repository_metadata.downloadable column. + for downloadable_revision in repository.metadata_revisions: metadata = downloadable_revision.metadata invalid_tools = metadata.get( 'invalid_tools', [] ) for invalid_tool_config in invalid_tools: @@ -410,7 +396,7 @@ .filter( and_( trans.model.Repository.table.c.deleted == False, trans.model.Repository.table.c.user_id == trans.user.id ) ) \ .order_by( trans.model.Repository.table.c.name ): - for downloadable_revision in repository.downloadable_revisions: + for downloadable_revision in repository.metadata_revisions: metadata = downloadable_revision.metadata invalid_tools = metadata.get( 'invalid_tools', [] ) for invalid_tool_config in invalid_tools: @@ -1595,7 +1581,7 @@ repo_dir = repository.repo_path repo = hg.repository( get_configured_ui(), repo_dir ) # Get the lower bound changeset revision - lower_bound_changeset_revision = get_previous_valid_changset_revision( repository, repo, changeset_revision ) + lower_bound_changeset_revision = get_previous_downloadable_changset_revision( repository, repo, changeset_revision ) # Build the list of changeset revision hashes. changeset_hashes = [] for changeset in reversed_lower_upper_bounded_changelog( repo, lower_bound_changeset_revision, changeset_revision ): diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/controllers/upload.py --- a/lib/galaxy/webapps/community/controllers/upload.py +++ b/lib/galaxy/webapps/community/controllers/upload.py @@ -130,9 +130,6 @@ error, error_message = handle_sample_tool_data_table_conf_file( trans.app, full_path ) if error: message = '%s<br/>%s' % ( message, error_message ) - #if full_path.endswith( '.loc.sample' ): - # # Handle the special case where a xxx.loc.sample file is being uploaded by copying it to ~/tool-data/xxx.loc. - # copy_sample_file( trans.app, full_path ) # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) @@ -272,7 +269,6 @@ if error: return False, message, files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed commands.commit( repo.ui, repo, full_path, user=trans.user.username, message=commit_message ) - # See if the content of the change set was valid. admin_only = len( repository.downloadable_revisions ) != 1 handle_email_alerts( trans, repository, content_alert_str=content_alert_str, new_repo_alert=new_repo_alert, admin_only=admin_only ) return True, '', files_to_remove, content_alert_str, undesirable_dirs_removed, undesirable_files_removed diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/model/mapping.py --- a/lib/galaxy/webapps/community/model/mapping.py +++ b/lib/galaxy/webapps/community/model/mapping.py @@ -121,7 +121,8 @@ Column( "changeset_revision", TrimmedString( 255 ), index=True ), Column( "metadata", JSONType, nullable=True ), Column( "tool_versions", JSONType, nullable=True ), - Column( "malicious", Boolean, default=False ) ) + Column( "malicious", Boolean, default=False ), + Column( "downloadable", Boolean, default=True ) ) RepositoryRatingAssociation.table = Table( "repository_rating_association", metadata, Column( "id", Integer, primary_key=True ), @@ -196,7 +197,11 @@ categories=relation( RepositoryCategoryAssociation ), ratings=relation( RepositoryRatingAssociation, order_by=desc( RepositoryRatingAssociation.table.c.update_time ), backref="repositories" ), user=relation( User.mapper ), - downloadable_revisions=relation( RepositoryMetadata, order_by=desc( RepositoryMetadata.table.c.update_time ) ) ) ) + downloadable_revisions=relation( RepositoryMetadata, + primaryjoin=( ( Repository.table.c.id == RepositoryMetadata.table.c.repository_id ) & ( RepositoryMetadata.table.c.downloadable == True ) ), + order_by=desc( RepositoryMetadata.table.c.update_time ) ), + metadata_revisions=relation( RepositoryMetadata, + order_by=desc( RepositoryMetadata.table.c.update_time ) ) ) ) assign_mapper( context, RepositoryMetadata, RepositoryMetadata.table, properties=dict( repository=relation( Repository ) ) ) diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/model/migrate/check.py --- a/lib/galaxy/webapps/community/model/migrate/check.py +++ b/lib/galaxy/webapps/community/model/migrate/check.py @@ -77,8 +77,10 @@ # Verify that the code and the DB are in sync db_schema = schema.ControlledSchema( engine, migrate_repository ) if migrate_repository.versions.latest != db_schema.version: - raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh upgrade'." - % ( db_schema.version, migrate_repository.versions.latest ) ) + exception_msg = "Your database has version '%d' but this code expects version '%d'. " % ( db_schema.version, migrate_repository.versions.latest ) + exception_msg += "Back up your database and then migrate the schema by running the following from your Galaxy installation directory:" + exception_msg += "\n\nsh manage_db.sh upgrade community\n" + raise Exception( exception_msg ) else: log.info( "At database version %d" % db_schema.version ) diff -r eff8b196858c1ff5a6227bd750a06ea924f8530c -r 0ffcc91708233cc087d9996235f0917b377cfe8a lib/galaxy/webapps/community/model/migrate/versions/0012_add_downloadable_column.py --- /dev/null +++ b/lib/galaxy/webapps/community/model/migrate/versions/0012_add_downloadable_column.py @@ -0,0 +1,48 @@ +""" +Migration script to add the downloadable column to the repository_metadata table. +""" + +from sqlalchemy import * +from sqlalchemy.orm import * +from migrate import * +from migrate.changeset import * + +import sys, logging +log = logging.getLogger( __name__ ) +log.setLevel(logging.DEBUG) +handler = logging.StreamHandler( sys.stdout ) +format = "%(name)s %(levelname)s %(asctime)s %(message)s" +formatter = logging.Formatter( format ) +handler.setFormatter( formatter ) +log.addHandler( handler ) + +metadata = MetaData( migrate_engine ) +db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) + +def upgrade(): + print __doc__ + metadata.reflect() + # Create and initialize imported column in job table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + c = Column( "downloadable", Boolean, default=True ) + try: + # Create + c.create( RepositoryMetadata_table ) + assert c is RepositoryMetadata_table.c.downloadable + # Initialize. + if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + default_true = "1" + elif migrate_engine.name == 'postgres': + default_true = "true" + db_session.execute( "UPDATE repository_metadata SET downloadable=%s" % default_true ) + except Exception, e: + print "Adding downloadable column to the repository_metadata table failed: %s" % str( e ) + +def downgrade(): + metadata.reflect() + # Drop downloadable column from repository_metadata table. + RepositoryMetadata_table = Table( "repository_metadata", metadata, autoload=True ) + try: + RepositoryMetadata_table.c.downloadable.drop() + except Exception, e: + print "Dropping column downloadable from the repository_metadata table failed: %s" % str( e ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.