2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/656e88fcfb70/ Changeset: 656e88fcfb70 User: dannon Date: 2015-01-30 21:17:27+00:00 Summary: Include sentry middleware in toolshed's build app. See TODO -- this will be refactored. Affected #: 1 file diff -r 3618b7a91e23096ecf7ce5842c1e16c749251a4d -r 656e88fcfb70edfd024ec0537630071676c33236 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -171,6 +171,14 @@ from paste import recursive app = recursive.RecursiveMiddleware( app, conf ) log.debug( "Enabling 'recursive' middleware" ) + # If sentry logging is enabled, log here before propogating up to + # the error middleware + # TODO sentry config is duplicated between tool_shed/galaxy, refactor this. + sentry_dsn = conf.get( 'sentry_dsn', None ) + if sentry_dsn: + from galaxy.web.framework.middleware.sentry import Sentry + log.debug( "Enabling 'sentry' middleware" ) + app = Sentry( app, sentry_dsn ) # Various debug middleware that can only be turned on if the debug # flag is set, either because they are insecure or greatly hurt # performance https://bitbucket.org/galaxy/galaxy-central/commits/9cd095c873a4/ Changeset: 9cd095c873a4 User: dannon Date: 2015-01-30 21:17:38+00:00 Summary: Merge. Affected #: 10 files diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 config/reports_wsgi.ini.sample --- a/config/reports_wsgi.ini.sample +++ b/config/reports_wsgi.ini.sample @@ -64,10 +64,10 @@ use_new_layout = true # Serving static files (needed if running standalone) -static_enabled = True -static_cache_time = 360 -static_dir = %(here)s/static/ -static_images_dir = %(here)s/static/images -static_favicon_dir = %(here)s/static/favicon.ico -static_scripts_dir = %(here)s/static/scripts/ -static_style_dir = %(here)s/static/june_2007_style/blue +# static_enabled = True +# static_cache_time = 360 +# static_dir = %(here)s/static/ +# static_images_dir = %(here)s/static/images +# static_favicon_dir = %(here)s/static/favicon.ico +# static_scripts_dir = %(here)s/static/scripts/ +# static_style_dir = %(here)s/static/june_2007_style/blue diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 eggs.ini --- a/eggs.ini +++ b/eggs.ini @@ -73,7 +73,7 @@ wsgiref = 0.1.2 Babel = 1.3 wchartype = 0.1 -Whoosh = 2.5.7 +Whoosh = 2.4.1 ; fluent_logger = 0.3.3 raven = 3.1.8 diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/util/__init__.py --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -371,7 +371,7 @@ if second_diff < 86400: return str(second_diff / 3600) + " hours ago" if day_diff == 1: - return "Yesterday" + return "yesterday" if day_diff < 7: return str( day_diff ) + " days ago" if day_diff < 31: @@ -381,16 +381,16 @@ return str( day_diff / 365 ) + " years ago" else: if day_diff == 0: - return "Today" + return "today" if day_diff == 1: - return "Yesterday" + return "yesterday" if day_diff < 7: - return "this week" + return "less than a week" if day_diff < 31: - return "this month" + return "less than a month" if day_diff < 365: - return "this year" - return str( day_diff / 365 ) + " years ago" + return "less than a year" + return "a few years ago" def pretty_print_json(json_data, is_json_string=False): diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/web/framework/webapp.py --- a/lib/galaxy/web/framework/webapp.py +++ b/lib/galaxy/web/framework/webapp.py @@ -814,3 +814,26 @@ template = Template( source=template_string, searchList=[context or kwargs, dict(caller=self)] ) return str(template) + + +def build_url_map( app, global_conf, local_conf ): + from paste.urlmap import URLMap + from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static + urlmap = URLMap() + # Merge the global and local configurations + conf = global_conf.copy() + conf.update(local_conf) + # Get cache time in seconds + cache_time = conf.get( "static_cache_time", None ) + if cache_time is not None: + cache_time = int( cache_time ) + # Send to dynamic app by default + urlmap["/"] = app + # Define static mappings from config + urlmap["/static"] = Static( conf.get( "static_dir", "./static/" ), cache_time ) + urlmap["/images"] = Static( conf.get( "static_images_dir", "./static/images" ), cache_time ) + urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir", "./static/scripts/" ), cache_time ) + urlmap["/static/style"] = Static( conf.get( "static_style_dir", "./static/style/blue" ), cache_time ) + urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir", "./static/favicon.ico" ), cache_time ) + urlmap["/robots.txt"] = Static( conf.get( "static_robots_txt", "./static/robots.txt" ), cache_time ) + return urlmap, cache_time diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/webapps/galaxy/api/provenance.py --- a/lib/galaxy/webapps/galaxy/api/provenance.py +++ b/lib/galaxy/webapps/galaxy/api/provenance.py @@ -52,15 +52,21 @@ if item.copied_from_library_dataset_dataset_association: item = item.copied_from_library_dataset_dataset_association job = item.creating_job - return { - "id": trans.security.encode_id(item.id), - "uuid": ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid), - "job_id": trans.security.encode_id( job.id ), - "tool_id": job.tool_id, - "parameters": self._get_job_record(trans, job, follow), - "stderr": job.stderr, - "stdout": job.stdout, - } + if job is not None: + return { + "id": trans.security.encode_id(item.id), + "uuid": ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid), + "job_id": trans.security.encode_id( job.id ), + "tool_id": job.tool_id, + "parameters": self._get_job_record(trans, job, follow), + "stderr": job.stderr, + "stdout": job.stdout, + } + else: + return { + "id": trans.security.encode_id(item.id), + "uuid": ( lambda uuid: str( uuid ) if uuid else None )( item.dataset.uuid) + } return None def _get_job_record(self, trans, job, follow): diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -610,26 +610,8 @@ return app def wrap_in_static( app, global_conf, plugin_frameworks=None, **local_conf ): - from paste.urlmap import URLMap from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static - urlmap = URLMap() - # Merge the global and local configurations - conf = global_conf.copy() - conf.update(local_conf) - # Get cache time in seconds - cache_time = conf.get( "static_cache_time", None ) - if cache_time is not None: - cache_time = int( cache_time ) - # Send to dynamic app by default - urlmap["/"] = app - # Define static mappings from config - urlmap["/static"] = Static( conf.get( "static_dir", "./static/" ), cache_time ) - urlmap["/images"] = Static( conf.get( "static_images_dir", "./static/images" ), cache_time ) - urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir", "./static/scripts/" ), cache_time ) - urlmap["/static/style"] = Static( conf.get( "static_style_dir", "./static/style/blue" ), cache_time ) - urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir", "./static/favicon.ico" ), cache_time ) - urlmap["/robots.txt"] = Static( conf.get( "static_robots_txt", "./static/robots.txt" ), cache_time ) - + urlmap, cache_time = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf ) # wrap any static dirs for plugins plugin_frameworks = plugin_frameworks or [] for framework in plugin_frameworks: diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/webapps/reports/buildapp.py --- a/lib/galaxy/webapps/reports/buildapp.py +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -62,7 +62,7 @@ # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) - if kwargs.get( 'static_enabled', True ): + if asbool( kwargs.get( 'static_enabled', True ) ): webapp = wrap_in_static( webapp, global_conf, **kwargs ) # Close any pooled database connections before forking try: @@ -135,25 +135,7 @@ return app def wrap_in_static( app, global_conf, **local_conf ): - from paste.urlmap import URLMap - from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static - urlmap = URLMap() - # Merge the global and local configurations - conf = global_conf.copy() - conf.update(local_conf) - # Get cache time in seconds - cache_time = conf.get( "static_cache_time", None ) - if cache_time is not None: - cache_time = int( cache_time ) - # Send to dynamic app by default - urlmap["/"] = app - # Define static mappings from config - urlmap["/static"] = Static( conf.get( "static_dir" ), cache_time ) - urlmap["/images"] = Static( conf.get( "static_images_dir" ), cache_time ) - urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir" ), cache_time ) - urlmap["/static/style"] = Static( conf.get( "static_style_dir" ), cache_time ) - urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) - # URL mapper becomes the root webapp + urlmap, _ = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf ) return urlmap def build_template_error_formatters(): diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/webapps/tool_shed/buildapp.py --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -134,7 +134,7 @@ # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) - if kwargs.get( 'static_enabled', True ): + if asbool( kwargs.get( 'static_enabled', True ) ): webapp = wrap_in_static( webapp, global_conf, **kwargs ) # Close any pooled database connections before forking try: @@ -227,26 +227,7 @@ return app def wrap_in_static( app, global_conf, **local_conf ): - from paste.urlmap import URLMap - from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static - urlmap = URLMap() - # Merge the global and local configurations - conf = global_conf.copy() - conf.update(local_conf) - # Get cache time in seconds - cache_time = conf.get( "static_cache_time", None ) - if cache_time is not None: - cache_time = int( cache_time ) - # Send to dynamic app by default - urlmap["/"] = app - # Define static mappings from config - urlmap["/static"] = Static( conf.get( "static_dir", "./static/" ), cache_time ) - urlmap["/images"] = Static( conf.get( "static_images_dir", "./static/images" ), cache_time ) - urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir", "./static/scripts/" ), cache_time ) - urlmap["/static/style"] = Static( conf.get( "static_style_dir", "./static/style/blue" ), cache_time ) - urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir", "./static/favicon.ico" ), cache_time ) - urlmap["/robots.txt"] = Static( conf.get( "static_robots_txt", "./static/robots.txt" ), cache_time ) - # URL mapper becomes the root webapp + urlmap, _ = galaxy.web.framework.webapp.build_url_map( app, global_conf, local_conf ) return urlmap def build_template_error_formatters(): diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 lib/galaxy/webapps/tool_shed/search/repo_search.py --- a/lib/galaxy/webapps/tool_shed/search/repo_search.py +++ b/lib/galaxy/webapps/tool_shed/search/repo_search.py @@ -45,7 +45,7 @@ use_final = True def final( self, searcher, docnum, score ): - log.debug('score before: ' + str(score) ) + # log.debug('score before: ' + str(score) ) # Arbitrary for now reasonable_hits = 100.0 @@ -59,15 +59,15 @@ if times_downloaded == 0: times_downloaded = 1 popularity_modifier = ( times_downloaded / reasonable_hits ) - log.debug('popularity_modifier: ' + str(popularity_modifier) ) + # log.debug('popularity_modifier: ' + str(popularity_modifier) ) cert_modifier = 2 if searcher.stored_fields( docnum )[ "approved" ] == 'yes' else 1 - log.debug('cert_modifier: ' + str(cert_modifier) ) + # log.debug('cert_modifier: ' + str(cert_modifier) ) # Adjust the computed score for this document by the popularity # and by the certification level. final_score = score * popularity_modifier * cert_modifier - log.debug('score after: ' + str( final_score ) ) + # log.debug('score after: ' + str( final_score ) ) return final_score @@ -108,16 +108,14 @@ 'remote_repository_url', 'repo_owner_username' ], schema = schema ) - # user_query = parser.parse( search_term ) user_query = parser.parse( '*' + search_term + '*' ) + hits = searcher.search_page( user_query, page, pagelen = 10, terms = True ) - # hits = searcher.search( user_query, terms = True ) - hits = searcher.search_page( user_query, page, pagelen = 10, terms = True ) log.debug( 'searching for: #' + str( search_term ) ) log.debug( 'total hits: ' + str( len( hits ) ) ) log.debug( 'scored hits: ' + str( hits.scored_length() ) ) results = {} - results[ 'total_results'] = str( hits.scored_length() ) + results[ 'total_results'] = str( len( hits ) ) results[ 'hits' ] = [] for hit in hits: hit_dict = {} diff -r 656e88fcfb70edfd024ec0537630071676c33236 -r 9cd095c873a411c56bbe8769393b92c3a2047e24 scripts/update_shed_config_path.py --- /dev/null +++ b/scripts/update_shed_config_path.py @@ -0,0 +1,80 @@ +import os +import argparse +import ConfigParser +import sys +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) +sys.path = new_path + +import logging +import galaxy.model.tool_shed_install +import galaxy.model.tool_shed_install.mapping as mapping +from galaxy.model.orm import * +from galaxy import eggs +eggs.require('sqlalchemy') +import sqlalchemy + +def main( opts, session, model ): + ''' + Find all tool shed repositories with the bad path and update with the correct path. + ''' + for row in session.query( model.ToolShedRepository ).all(): + if 'shed_config_filename' in row.metadata: + if row.metadata['shed_config_filename'] == opts.bad_filename: + row.metadata['shed_config_filename'] = opts.good_filename + session.add( row ) + session.flush() + return 0 + +def create_database( config_file ): + parser = ConfigParser.SafeConfigParser() + parser.read( config_file ) + # Determine which database connection to use. + database_connection = parser.get( 'app:main', 'install_database_connection' ) + if database_connection is None: + database_connection = parser.get( 'app:main', 'database_connection' ) + if database_connection is None: + database_connection = 'sqlite:///%s' % parser.get( 'app:main', 'database_file' ) + if database_connection is None: + print 'Unable to determine correct database connection.' + exit(1) + + '''Initialize the database file.''' + dialect_to_egg = { + "sqlite" : "pysqlite>=2", + "postgres" : "psycopg2", + "postgresql" : "psycopg2", + "mysql" : "MySQL_python" + } + dialect = ( database_connection.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[ dialect ] + try: + eggs.require( egg ) + print( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + print( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support. + print( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + # Initialize the database connection. + engine = create_engine( database_connection ) + meta = MetaData( bind=engine ) + install_session = Session = scoped_session( sessionmaker( bind=engine, autoflush=False, autocommit=True ) ) + model = mapping.init( database_connection ) + return install_session, model + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( '--config_file', dest='config_file', required=True, help="The path to your Galaxy configuration .ini file." ) + parser.add_argument( '--from', dest='bad_filename', required=True, help="The old, invalid path to the shed_tool_conf.xml or migrated_tools_conf.xml file." ) + parser.add_argument( '--to', dest='good_filename', required=True, help="The updated path to the shed_tool_conf.xml or migrated_tools_conf.xml file." ) + parser.add_argument( '--force', dest='force', action='store_true', help="Use this flag to set the new path even if the file does not (yet) exist there." ) + opts = parser.parse_args() + if not os.path.exists( opts.good_filename ) and not opts.force: + print 'The file %s does not exist, use the --force option to proceed.' % opts.good_filename + exit(1) + session, model = create_database( opts.config_file ) + exit( main( opts, session, model ) ) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.