- galaxy-commits - lists.galaxyproject.org

commit/galaxy-central: inithello: Refactor summary output. Improve handling of empty tool_test_errors dict. Refactor repository information retrieval.
by commits-noreply＠bitbucket.org 11 Mar '13

11 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0a6b63eca3ff/ changeset: 0a6b63eca3ff user: inithello date: 2013-03-11 05:47:47 summary: Refactor summary output. Improve handling of empty tool_test_errors dict. Refactor repository information retrieval. affected #: 2 files diff -r 2b4d6af45a04ddb49c2c87a0ead3874503d35832 -r 0a6b63eca3ff6a72cd874bf36f73a2b5905aff65 install_and_test_tool_shed_repositories.sh --- a/install_and_test_tool_shed_repositories.sh +++ b/install_and_test_tool_shed_repositories.sh @@ -1,7 +1,6 @@ #!/bin/sh # A good place to look for nose info: http://somethingaboutorange.com/mrl/projects/nose/ -#rm -f ./test/tool_shed/run_functional_tests.log -python test/install_and_test_tool_shed_repositories/functional_tests.py -v --with-nosehtml --html-report-file ./test/install_and_test_tool_shed_repositories/run_functional_tests.html test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py test/functional/test_toolbox.py +python test/install_and_test_tool_shed_repositories/functional_tests.py $* -v --with-nosehtml --html-report-file ./test/install_and_test_tool_shed_repositories/run_functional_tests.html test/install_and_test_tool_shed_repositories/functional/test_install_repositories.py test/functional/test_toolbox.py diff -r 2b4d6af45a04ddb49c2c87a0ead3874503d35832 -r 0a6b63eca3ff6a72cd874bf36f73a2b5905aff65 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -242,6 +242,8 @@ api_path = metadata_dict[ 'url' ].split( '/' ) api_url = get_api_url( base=tool_shed_url, parts=api_path ) repository_metadata = json_from_url( api_url ) + if repository_metadata[ 'tool_test_errors' ] is None: + return {} return repository_metadata[ 'tool_test_errors' ] def json_from_url( url ): @@ -276,16 +278,22 @@ result = test_runner.run( tests ) return result, test_config.plugins._plugins +def show_summary_output( repository_info_dicts ): + repositories_by_owner = dict() + for repository in repository_info_dicts: + if repository[ 'owner' ] not in repositories_by_owner: + repositories_by_owner[ repository[ 'owner' ] ] = [] + repositories_by_owner[ repository[ 'owner' ] ].append( repository ) + for owner in repositories_by_owner: + print "# " + for repository in repositories_by_owner[ owner ]: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + def main(): # ---- Configuration ------------------------------------------------------ galaxy_test_host = os.environ.get( 'GALAXY_INSTALL_TEST_HOST', default_galaxy_test_host ) galaxy_test_port = os.environ.get( 'GALAXY_INSTALL_TEST_PORT', str( default_galaxy_test_port_max ) ) - # Initialize some variables for the summary that will be printed to stdout. - repositories_tested = 0 - repositories_passed = [] - repositories_failed = [] - tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: os.environ[ 'HTTP_ACCEPT_LANGUAGE' ] = default_galaxy_locales @@ -444,38 +452,19 @@ log.info( "The embedded Galaxy application is running on %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) success = False + # Initialize some variables for the summary that will be printed to stdout. + repositories_tested = 0 + repositories_passed = [] + repositories_failed = [] + repositories_failed_install = [] try: - # Iterate through a list of repository info dicts. + detailed_repository_list = [] + # Get a list of repositories to test from the tool shed specified in the GALAXY_INSTALL_TEST_TOOL_SHED_URL environment variable. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) repositories_to_install = get_repositories_to_install( galaxy_tool_shed_url, source='url' ) log.info( "Retrieved %d repositories to install..." % len( repositories_to_install ) ) repositories_tested = len( repositories_to_install ) - # This loop will iterate through the list of repositories returned by the above method, skipping any that are marked - # as deleted. For each repository, it will generate a test method that will use Twill to install that repository into the - # embedded Galaxy application that was started up, selecting to install repository and tool dependencies if they are - # defined. If the installation completes successfully, it will then generate a test case for each functional test - # defined for each tool in the repository, and execute the generated test cases. When this completes, it will record - # the result of the tests, and if any failed, the traceback and captured output of the tool that was run. - # After all tests have completed, the repository is uninstalled, so that the previous test cases don't interfere with - # the next repository's functional tests. for repository_to_install_dict in repositories_to_install: - """ - Each repository_to_install_dict looks something like: - { - "changeset_revision": "13fa22a258b5", - "downloadable": true, - "id": "529fd61ab1c6cc36", - "malicious": false, - "repository_id": "529fd61ab1c6cc36", - "url": "/api/repository_revisions/529fd61ab1c6cc36" - } - """ - repository_status = dict() - repository_id = repository_to_install_dict.get( 'repository_id', None ) - changeset_revision = repository_to_install_dict.get( 'changeset_revision', None ) - metadata_revision_id = repository_to_install_dict.get( 'id', None ) - # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. - repository_to_install_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url # We need to get some details from the tool shed API, such as repository name and owner, to pass on to the # module that will generate the install methods. repository_info_dict = get_repository_info_from_api( galaxy_tool_shed_url, repository_to_install_dict ) @@ -483,13 +472,31 @@ # and functionally correct tools that someone has previously installed. Deleted repositories have never been installed, # and therefore do not need to be checked. If they are undeleted, this script will then test them the next time it runs. if repository_info_dict[ 'deleted' ]: - log.info( "Skipping revision %s of repository id %s since the repository is deleted..." % ( str( changeset_revision ), str( repository_id ) ) ) + log.info( "Skipping revision %s of repository id %s (%s/%s) since the repository is deleted..." % \ + ( repository_info_dict[ 'changeset_revision' ], + repository_info_dict[ 'repository_id' ], + repository_info_dict[ 'owner' ], + repository_info_dict[ 'name' ] ) ) continue - log.info( "Installing and testing revision %s of repository id %s..." % ( str( changeset_revision ), str( repository_id ) ) ) - # Add repository details to the basic repository dict. - repository_dict = dict( repository_info_dict.items() + repository_to_install_dict.items() ) + # Now merge the dict returned from /api/repository_revisions with the detailed dict we just retrieved. + detailed_repository_list.append( dict( repository_info_dict.items() + repository_to_install_dict.items() ) ) + if '-list_repositories' in sys.argv: + log.info( "The API returned the following repositories, not counting deleted:" ) + for repository_info_dict in detailed_repository_list: + log.info( "%s owned by %s changeset revision %s" % ( repository_info_dict.get( 'name', None ), + repository_info_dict.get( 'owner', None ), + repository_info_dict.get( 'changeset_revision', None ) ) ) + # This loop will iterate through the list of repositories generated by the above code, having already filtered out any + # that were marked as deleted. For each repository, it will generate a test method that will use Twill to install that + # repository into the embedded Galaxy application that was started up, selecting to install repository and tool + # dependencies if they are defined. If the installation completes successfully, it will then generate a test case for + # each functional test defined for each tool in the repository, and execute the generated test cases. When this completes, + # it will record the result of the tests, and if any failed, the traceback and captured output of the tool that was run. + # After all tests have completed, the repository is uninstalled, so that the previous test cases don't interfere with + # the next repository's functional tests. + for repository_info_dict in detailed_repository_list: """ - After the addition of the repository details, each repository_dict should now contain something like: + Each repository_info_dict looks something like: { "changeset_revision": "13fa22a258b5", "contents_url": "/api/repositories/529fd61ab1c6cc36/contents", @@ -510,11 +517,19 @@ "user_id": "529fd61ab1c6cc36" } """ - name = repository_dict[ 'name' ] - owner = repository_dict[ 'owner' ] + repository_status = dict() + repository_id = repository_info_dict.get( 'repository_id', None ) + changeset_revision = repository_info_dict.get( 'changeset_revision', None ) + metadata_revision_id = repository_info_dict.get( 'id', None ) + # Add the URL for the tool shed we're installing from, so the automated installation methods go to the right place. + repository_info_dict[ 'tool_shed_url' ] = galaxy_tool_shed_url + log.info( "Installing and testing revision %s of repository id %s..." % ( str( changeset_revision ), str( repository_id ) ) ) + # Get the name and owner out of the repository info dict. + name = repository_info_dict[ 'name' ] + owner = repository_info_dict[ 'owner' ] # Use the repository information dict to generate an install method that will install the repository into the embedded # Galaxy application, with tool dependencies and repository dependencies, if any. - test_install_repositories.generate_install_method( repository_dict ) + test_install_repositories.generate_install_method( repository_info_dict ) os.environ[ 'GALAXY_INSTALL_TEST_HOST' ] = galaxy_test_host # Configure nose to run the install method as a test. test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) @@ -526,7 +541,7 @@ # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to # sh run_functional_tests.sh -installed if success: - log.debug( 'Installation of %s succeeded, running all defined functional tests.' % repository_dict[ 'name' ] ) + log.debug( 'Installation of %s succeeded, running all defined functional tests.' % name ) # Generate the shed_tools_dict that specifies the location of test data contained within this repository. If the repository # does not have a test-data directory, this will return has_test_data = False, and we will set the do_not_test flag to True, # and the tools_functionally_correct flag to False, as well as updating tool_test_errors. @@ -574,17 +589,18 @@ # }, # ] # } - repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, repository_dict ) + repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, repository_info_dict ) + if 'test_environment' not in repository_status: + repository_status[ 'test_environment' ] = {} test_environment = get_test_environment( repository_status[ 'test_environment' ] ) test_environment[ 'galaxy_database_version' ] = get_database_version( app ) test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) repository_status[ 'test_environment' ] = test_environment repository_status[ 'tests_passed' ] = [] repository_status[ 'test_errors' ] = [] + repository_status[ 'invalid_tests' ] = [] if not has_test_data: log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) - repository_status[ 'test_environment' ] = get_test_environment() - test_id = 'Find functional test data for %s' % repository_dict[ 'name' ] # Record the lack of test data. test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) @@ -593,7 +609,7 @@ register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. - execute_uninstall_method( repository_dict ) + execute_uninstall_method( repository_info_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) @@ -618,9 +634,7 @@ # Run the configured tests. result, test_plugins = run_tests( test_config ) success = result.wasSuccessful() - # Record some information about the environment in which this test was run, in case a failure is specific to a certain processor - # architecture or operating system. - repository_dict[ 'test_environment' ] = test_environment + # Use the ReportResults nose plugin to get a list of tests that passed. for plugin in test_plugins: if hasattr( plugin, 'getTestStatus' ): test_identifier = '%s/%s' % ( owner, name ) @@ -637,10 +651,8 @@ # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) - repository_status[ 'test_environment' ] = test_environment register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) - log.debug( 'Revision %s of repository %s installed and passed functional tests.' % \ - ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) + log.debug( 'Revision %s of repository %s installed and passed functional tests.' % ( changeset_revision, name ) ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. for failure in result.failures: @@ -686,14 +698,15 @@ repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ - ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) + ( changeset_revision, name ) ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. - execute_uninstall_method( repository_dict ) + execute_uninstall_method( repository_info_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox else: - log.debug( 'Repository %s failed to install correctly.' % repository_dict[ 'name' ] ) + repositories_failed_install.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + log.debug( 'Repository %s failed to install correctly.' % name ) except: log.exception( "Failure running tests" ) @@ -728,29 +741,17 @@ print "# Repository revisions tested: %d" % repositories_tested if repositories_tested > 0: if repositories_passed: - passed_repositories_by_owner = dict() - for repository in repositories_passed: - if repository[ 'owner' ] not in passed_repositories_by_owner: - passed_repositories_by_owner[ repository[ 'owner' ] ] = [] - passed_repositories_by_owner[ repository[ 'owner' ] ].append( repository ) print "# " print "# Repositories passed:" - for owner in passed_repositories_by_owner: - print "# " - for repository in passed_repositories_by_owner[ owner ]: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + show_summary_output( repositories_passed ) if repositories_failed: - failed_repositories_by_owner = dict() - for repository in repositories_failed: - if repository[ 'owner' ] not in failed_repositories_by_owner: - failed_repositories_by_owner[ repository[ 'owner' ] ] = [] - failed_repositories_by_owner[ repository[ 'owner' ] ].append( repository ) print "# " print "# Repositories failed:" - for owner in failed_repositories_by_owner: - print "# " - for repository in failed_repositories_by_owner[ owner ]: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + show_summary_output( repositories_failed ) + if repositories_failed_install: + print "# " + print "# Repositories not installed correctly:" + show_summary_output( repositories_failed_install ) print "####################################################################################" if success: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Explicitly clear ReportResults.passed after each set of tests. Group test results by repository owner, and separate with a blank line.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/2b4d6af45a04/ changeset: 2b4d6af45a04 user: inithello date: 2013-03-08 23:03:32 summary: Explicitly clear ReportResults.passed after each set of tests. Group test results by repository owner, and separate with a blank line. affected #: 1 file diff -r f25f3fee4da76fc0660ba27a1b26673d7592ddc1 -r 2b4d6af45a04ddb49c2c87a0ead3874503d35832 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -120,7 +120,7 @@ class ReportResults( Plugin ): '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' name = "reportresults" - passed = [] + passed = dict() def options( self, parser, env=os.environ ): super( ReportResults, self ).options( parser, env=env ) @@ -137,10 +137,18 @@ # Rearrange the test ID to match the format that is produced in test_results.failures test_id_parts = test_id.split( '.' ) fixed_test_id = '%s (%s)' % ( test_id_parts[ -1 ], '.'.join( test_id_parts[ :-1 ] ) ) - self.passed.append( fixed_test_id ) + test_parts = fixed_test_id.split( '/' ) + owner = test_parts[ -4 ] + name = test_parts[ -3 ] + test_identifier = '%s/%s' % ( owner, name ) + if test_identifier not in self.passed: + self.passed[ test_identifier ] = [] + self.passed[ test_identifier ].append( fixed_test_id ) - def getTestStatus( self ): - return self.passed + def getTestStatus( self, test_identifier ): + tests_passed = self.passed[ test_identifier ] + del self.passed[ test_identifier ] + return tests_passed def execute_uninstall_method( repository_dict ): # Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them @@ -615,7 +623,8 @@ repository_dict[ 'test_environment' ] = test_environment for plugin in test_plugins: if hasattr( plugin, 'getTestStatus' ): - tests_passed = plugin.getTestStatus() + test_identifier = '%s/%s' % ( owner, name ) + tests_passed = plugin.getTestStatus( test_identifier ) break repository_status[ 'tests_passed' ] = [] for test_id in tests_passed: @@ -670,7 +679,6 @@ if output_type in tmp_output: test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) repository_status[ 'test_errors' ].append( test_status ) - log.debug( to_json_string( repository_status, indent=2, sort_keys=True ) ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome # of the tests, and updates tool_test_errors with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested @@ -720,13 +728,29 @@ print "# Repository revisions tested: %d" % repositories_tested if repositories_tested > 0: if repositories_passed: + passed_repositories_by_owner = dict() + for repository in repositories_passed: + if repository[ 'owner' ] not in passed_repositories_by_owner: + passed_repositories_by_owner[ repository[ 'owner' ] ] = [] + passed_repositories_by_owner[ repository[ 'owner' ] ].append( repository ) + print "# " print "# Repositories passed:" - for repository in repositories_passed: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + for owner in passed_repositories_by_owner: + print "# " + for repository in passed_repositories_by_owner[ owner ]: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) if repositories_failed: + failed_repositories_by_owner = dict() + for repository in repositories_failed: + if repository[ 'owner' ] not in failed_repositories_by_owner: + failed_repositories_by_owner[ repository[ 'owner' ] ] = [] + failed_repositories_by_owner[ repository[ 'owner' ] ].append( repository ) + print "# " print "# Repositories failed:" - for repository in repositories_failed: - print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + for owner in failed_repositories_by_owner: + print "# " + for repository in failed_repositories_by_owner[ owner ]: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) print "####################################################################################" if success: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove unused yet exposed 'stdoutX' method of the datasets controller.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f25f3fee4da7/ changeset: f25f3fee4da7 user: dannon date: 2013-03-08 22:01:25 summary: Remove unused yet exposed 'stdoutX' method of the datasets controller. affected #: 1 file diff -r 619995bcb99feda43d9a369a7aa563108a19f3c7 -r f25f3fee4da76fc0660ba27a1b26673d7592ddc1 lib/galaxy/webapps/galaxy/controllers/dataset.py --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -180,16 +180,6 @@ if not hda or not self._can_access_dataset( trans, hda ): return trans.show_error_message( "Either this dataset does not exist or you do not have permission to access it." ) return trans.fill_template( "dataset/errors.mako", hda=hda ) - @web.expose - def stdoutX( self, trans, dataset_id=None, **kwargs ): - trans.response.set_content_type( 'text/plain' ) - try: - hda = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( dataset_id ) ) - assert hda and self._can_access_dataset( trans, hda ) - job = hda.creating_job_associations[0].job - except: - return "Invalid dataset ID or you are not allowed to access this dataset" - return job.stdout @web.expose def stdout( self, trans, dataset_id=None, **kwargs ): Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Enhance the script that checks repositories for functional tests, adding support for multiple web frontends.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/619995bcb99f/ changeset: 619995bcb99f user: inithello date: 2013-03-08 20:44:57 summary: Enhance the script that checks repositories for functional tests, adding support for multiple web frontends. affected #: 1 file diff -r c937f80188d8026f2074f8a897e2ff2972d8da6d -r 619995bcb99feda43d9a369a7aa563108a19f3c7 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -38,6 +38,12 @@ '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) + parser.add_option( "-s", + "--section", + action="store", + dest="section", + default='server:main', + help="which .ini file section to extract the host and port from" ) parser.add_option( "-v", "--verbose", action="count", dest="verbosity", @@ -57,10 +63,11 @@ config_dict[key] = value config = tool_shed_config.Configuration( **config_dict ) + config_section = options.section now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" print "# %s - Checking repositories for tools with functional tests." % now - print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( 'server:main', 'host' ), config_parser.get( 'server:main', 'port' ) ) + print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( config_section, 'host' ), config_parser.get( config_section, 'port' ) ) app = FlagRepositoriesApplication( config ) if options.info_only: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 4 new changesets
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

4 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/8e00d2fd674a/ changeset: 8e00d2fd674a user: jmchilton date: 2013-02-10 17:16:26 summary: Breakup big get_path_paste_uploaded_datasets function in library_common.py into smaller more extensible pieces. This is useful in implementing multiple file dataset uploads downstream but is also a good refactoring on its own. affected #: 1 file diff -r 506484344db3a370f8ae24096041d38557d1967e -r 8e00d2fd674acafb36e7118afaaf7f739235be2e lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1075,6 +1075,7 @@ return output def make_library_uploaded_dataset( self, trans, cntrller, params, name, path, type, library_bunch, in_folder=None ): link_data_only = params.get( 'link_data_only', 'copy_files' ) + file_type = params.file_type library_bunch.replace_dataset = None # not valid for these types of upload uploaded_dataset = util.bunch.Bunch() new_name = name @@ -1089,7 +1090,7 @@ uploaded_dataset.path = path uploaded_dataset.type = type uploaded_dataset.ext = None - uploaded_dataset.file_type = params.file_type + uploaded_dataset.file_type = file_type uploaded_dataset.dbkey = params.dbkey uploaded_dataset.space_to_tab = params.space_to_tab if in_folder: @@ -1145,44 +1146,57 @@ uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) return uploaded_datasets, 200, None def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, response_code, message ): + preserve_dirs = util.string_as_bool( params.get( 'preserve_dirs', False ) ) + uploaded_datasets = [] + (files_and_folders, _response_code, _message) = self._get_path_files_and_folders(params, preserve_dirs) + if _response_code: + return (uploaded_datasets, _response_code, _message) + for (path, name, folder) in files_and_folders: + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, path, 'path_paste', library_bunch, folder ) ) + return uploaded_datasets, 200, None + + def _get_path_files_and_folders( self, params, preserve_dirs ): + problem_response = self._check_path_paste_params( params ) + if problem_response: + return problem_response + files_and_folders = [] + for (line, path) in self._paths_list( params ): + line_files_and_folders = self._get_single_path_files_and_folders( line, path, preserve_dirs ) + files_and_folders.extend( line_files_and_folders ) + return files_and_folders, None, None + + def _get_single_path_files_and_folders(self, line, path, preserve_dirs): + files_and_folders = [] + if os.path.isfile( path ): + name = os.path.basename( path ) + files_and_folders.append((path, name, None)) + for basedir, dirs, files in os.walk( line ): + for file in files: + file_path = os.path.abspath( os.path.join( basedir, file ) ) + if preserve_dirs: + in_folder = os.path.dirname( file_path.replace( path, '', 1 ).lstrip( '/' ) ) + else: + in_folder = None + files_and_folders.append((file_path, file, in_folder)) + return files_and_folders + def _paths_list(self, params): + return [ (l.strip(), os.path.abspath(l.strip())) for l in params.filesystem_paths.splitlines() if l.strip() ] + + def _check_path_paste_params(self, params): if params.get( 'filesystem_paths', '' ) == '': message = "No paths entered in the upload form" response_code = 400 return None, response_code, message - preserve_dirs = util.string_as_bool( params.get( 'preserve_dirs', False ) ) - # locate files bad_paths = [] - uploaded_datasets = [] - for line in [ l.strip() for l in params.filesystem_paths.splitlines() if l.strip() ]: - path = os.path.abspath( line ) + for (_, path) in self._paths_list( params ): if not os.path.exists( path ): bad_paths.append( path ) - continue - # don't bother processing if we're just going to return an error - if not bad_paths: - if os.path.isfile( path ): - name = os.path.basename( path ) - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, path, 'path_paste', library_bunch ) ) - for basedir, dirs, files in os.walk( line ): - for file in files: - file_path = os.path.abspath( os.path.join( basedir, file ) ) - if preserve_dirs: - in_folder = os.path.dirname( file_path.replace( path, '', 1 ).lstrip( '/' ) ) - else: - in_folder = None - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, - cntrller, - params, - file, - file_path, - 'path_paste', - library_bunch, - in_folder ) ) if bad_paths: message = "Invalid paths:<br><ul><li>%s</li></ul>" % "</li><li>".join( bad_paths ) response_code = 400 return None, response_code, message - return uploaded_datasets, 200, None + return None + @web.expose def add_history_datasets_to_library( self, trans, cntrller, library_id, folder_id, hda_ids='', **kwd ): params = util.Params( kwd ) https://bitbucket.org/galaxy/galaxy-central/commits/c85bf30a5c35/ changeset: c85bf30a5c35 user: jmchilton date: 2013-02-10 17:16:27 summary: Breakup big get_server_dir_uploaded_datasets function in library_common.py into smaller more extensible pieces. This is useful in implementing multiple file dataset uploads downstream but is also a good refactoring on its own. affected #: 1 file diff -r 8e00d2fd674acafb36e7118afaaf7f739235be2e -r c85bf30a5c35a7fab80d9499b0852eeb2e433039 lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1106,6 +1106,16 @@ trans.sa_session.flush() return uploaded_dataset def get_server_dir_uploaded_datasets( self, trans, cntrller, params, full_dir, import_dir_desc, library_bunch, response_code, message ): + dir_response = self._get_server_dir_files(params, full_dir, import_dir_desc) + files = dir_response[0] + if not files: + return dir_response + uploaded_datasets = [] + for file in files: + name = os.path.basename( file ) + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) + return uploaded_datasets, 200, None + def _get_server_dir_files( self, params, full_dir, import_dir_desc ): files = [] try: for entry in os.listdir( full_dir ): @@ -1140,11 +1150,7 @@ message = "The directory '%s' contains no valid files" % full_dir response_code = 400 return None, response_code, message - uploaded_datasets = [] - for file in files: - name = os.path.basename( file ) - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) - return uploaded_datasets, 200, None + return files, None, None def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, response_code, message ): preserve_dirs = util.string_as_bool( params.get( 'preserve_dirs', False ) ) uploaded_datasets = [] https://bitbucket.org/galaxy/galaxy-central/commits/e1e8ddf3401a/ changeset: e1e8ddf3401a user: dannon date: 2013-03-08 20:32:34 summary: Merge pull request 108, https://bitbucket.org/galaxy/galaxy-central/pull-request/108/collaborate-on… affected #: 1 file diff -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b -r e1e8ddf3401ab1e028fbebd9fa8f4307b8d3ae03 lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1077,6 +1077,7 @@ def make_library_uploaded_dataset( self, trans, cntrller, params, name, path, type, library_bunch, in_folder=None ): link_data_only = params.get( 'link_data_only', 'copy_files' ) uuid_str = params.get( 'uuid', None ) + file_type = params.file_type library_bunch.replace_dataset = None # not valid for these types of upload uploaded_dataset = util.bunch.Bunch() new_name = name @@ -1091,7 +1092,7 @@ uploaded_dataset.path = path uploaded_dataset.type = type uploaded_dataset.ext = None - uploaded_dataset.file_type = params.file_type + uploaded_dataset.file_type = file_type uploaded_dataset.dbkey = params.dbkey uploaded_dataset.space_to_tab = params.space_to_tab if in_folder: @@ -1108,6 +1109,16 @@ trans.sa_session.flush() return uploaded_dataset def get_server_dir_uploaded_datasets( self, trans, cntrller, params, full_dir, import_dir_desc, library_bunch, response_code, message ): + dir_response = self._get_server_dir_files(params, full_dir, import_dir_desc) + files = dir_response[0] + if not files: + return dir_response + uploaded_datasets = [] + for file in files: + name = os.path.basename( file ) + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) + return uploaded_datasets, 200, None + def _get_server_dir_files( self, params, full_dir, import_dir_desc ): files = [] try: for entry in os.listdir( full_dir ): @@ -1142,50 +1153,59 @@ message = "The directory '%s' contains no valid files" % full_dir response_code = 400 return None, response_code, message + return files, None, None + def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, response_code, message ): + preserve_dirs = util.string_as_bool( params.get( 'preserve_dirs', False ) ) uploaded_datasets = [] - for file in files: - name = os.path.basename( file ) - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, file, 'server_dir', library_bunch ) ) + (files_and_folders, _response_code, _message) = self._get_path_files_and_folders(params, preserve_dirs) + if _response_code: + return (uploaded_datasets, _response_code, _message) + for (path, name, folder) in files_and_folders: + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, path, 'path_paste', library_bunch, folder ) ) return uploaded_datasets, 200, None - def get_path_paste_uploaded_datasets( self, trans, cntrller, params, library_bunch, response_code, message ): + + def _get_path_files_and_folders( self, params, preserve_dirs ): + problem_response = self._check_path_paste_params( params ) + if problem_response: + return problem_response + files_and_folders = [] + for (line, path) in self._paths_list( params ): + line_files_and_folders = self._get_single_path_files_and_folders( line, path, preserve_dirs ) + files_and_folders.extend( line_files_and_folders ) + return files_and_folders, None, None + + def _get_single_path_files_and_folders(self, line, path, preserve_dirs): + files_and_folders = [] + if os.path.isfile( path ): + name = os.path.basename( path ) + files_and_folders.append((path, name, None)) + for basedir, dirs, files in os.walk( line ): + for file in files: + file_path = os.path.abspath( os.path.join( basedir, file ) ) + if preserve_dirs: + in_folder = os.path.dirname( file_path.replace( path, '', 1 ).lstrip( '/' ) ) + else: + in_folder = None + files_and_folders.append((file_path, file, in_folder)) + return files_and_folders + def _paths_list(self, params): + return [ (l.strip(), os.path.abspath(l.strip())) for l in params.filesystem_paths.splitlines() if l.strip() ] + + def _check_path_paste_params(self, params): if params.get( 'filesystem_paths', '' ) == '': message = "No paths entered in the upload form" response_code = 400 return None, response_code, message - preserve_dirs = util.string_as_bool( params.get( 'preserve_dirs', False ) ) - # locate files bad_paths = [] - uploaded_datasets = [] - for line in [ l.strip() for l in params.filesystem_paths.splitlines() if l.strip() ]: - path = os.path.abspath( line ) + for (_, path) in self._paths_list( params ): if not os.path.exists( path ): bad_paths.append( path ) - continue - # don't bother processing if we're just going to return an error - if not bad_paths: - if os.path.isfile( path ): - name = os.path.basename( path ) - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, cntrller, params, name, path, 'path_paste', library_bunch ) ) - for basedir, dirs, files in os.walk( line ): - for file in files: - file_path = os.path.abspath( os.path.join( basedir, file ) ) - if preserve_dirs: - in_folder = os.path.dirname( file_path.replace( path, '', 1 ).lstrip( '/' ) ) - else: - in_folder = None - uploaded_datasets.append( self.make_library_uploaded_dataset( trans, - cntrller, - params, - file, - file_path, - 'path_paste', - library_bunch, - in_folder ) ) if bad_paths: message = "Invalid paths:<br><ul><li>%s</li></ul>" % "</li><li>".join( bad_paths ) response_code = 400 return None, response_code, message - return uploaded_datasets, 200, None + return None + @web.expose def add_history_datasets_to_library( self, trans, cntrller, library_id, folder_id, hda_ids='', **kwd ): params = util.Params( kwd ) https://bitbucket.org/galaxy/galaxy-central/commits/c937f80188d8/ changeset: c937f80188d8 user: dannon date: 2013-03-08 20:41:10 summary: Merge affected #: 3 files diff -r e1e8ddf3401ab1e028fbebd9fa8f4307b8d3ae03 -r c937f80188d8026f2074f8a897e2ff2972d8da6d lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -38,7 +38,16 @@ '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) - parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", help="verbose mode, print the name, owner, and changeset revision of each repository", default=False ) + parser.add_option( + "-v", "--verbose", + action="count", dest="verbosity", + default=1, + help="Control the amount of detail in the log output.") + parser.add_option( + "--verbosity", action="store", dest="verbosity", + metavar='VERBOSITY', + type="int", help="Control the amount of detail in the log output. --verbosity=1 is " + "the same as -v") ( options, args ) = parser.parse_args() ini_file = args[0] config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) @@ -56,12 +65,12 @@ if options.info_only: print "# Displaying info only ( --info_only )" - if options.verbose: - print "# Displaying extra information ( --verbose )" + if options.verbosity: + print "# Displaying extra information ( --verbosity = %d )" % options.verbosity - check_and_flag_repositories( app, info_only=options.info_only, verbose=options.verbose ) + check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) -def check_and_flag_repositories( app, info_only=False, verbose=False ): +def check_and_flag_repositories( app, info_only=False, verbosity=1 ): ''' This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, then checking the tool metadata for tests. @@ -97,10 +106,15 @@ and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute with the following structure: + If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute following this structure: { "test_environment": { + "galaxy_revision": "9001:abcd1234", + "galaxy_database_version": "114", + "tool_shed_revision": "9001:abcd1234", + "tool_shed_mercurial_version": "2.3.1", + "tool_shed_database_version": "17", "python_version": "2.7.2", "architecture": "x86_64", "system": "Darwin 12.2.0" @@ -108,8 +122,28 @@ "test_errors": [ { - "test_id": "Something that will easily identify what the problem is", - "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + "test_id": "The test ID, generated by twill", + "tool_id": "The tool ID that was tested", + "tool_version": "The tool version that was tested", + "stderr": "The output of the test, or a more detailed description of what was tested and what the error was." + "traceback": "The traceback, if any." + }, + ] + "passed_tests": + [ + { + "test_id": "The test ID, generated by twill", + "tool_id": "The tool ID that was tested", + "tool_version": "The tool version that was tested", + }, + ] + "invalid_tests": + [ + { + "tool_id": "The tool ID that does not have functional tests defined.", + "tool_version": "The version of the tool." + "tool_guid": "The guid of the tool." + "reason_test_is_invalid": "A short explanation of what is invalid. }, ] } @@ -129,19 +163,24 @@ repository_status[ 'test_environment' ][ 'tool_shed_database_version' ] = get_database_version( app ) repository_status[ 'test_environment' ][ 'tool_shed_mercurial_version' ] = __version__.version repository_status[ 'test_environment' ][ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_errors' ] = [] + repository_status[ 'invalid_tests' ] = [] metadata_records_to_check = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, app.model.RepositoryMetadata.table.c.do_not_test == False, app.model.RepositoryMetadata.table.c.tools_functionally_correct == False ) ) \ .all() for metadata_record in metadata_records_to_check: - repository_status[ 'test_errors' ] = [] + if metadata_record.tool_test_errors: + repository_status = metadata_record.tool_test_errors name = metadata_record.repository.name owner = metadata_record.repository.user.username changeset_revision = str( metadata_record.changeset_revision ) + repository_status[ 'invalid_tests' ] = [] if metadata_record.repository.id not in checked_repository_ids: checked_repository_ids.append( metadata_record.repository.id ) + if verbosity >= 1: + print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ + ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: @@ -152,19 +191,26 @@ for tool_metadata in metadata_record.metadata[ 'tools' ]: tool_count += 1 tool_id = tool_metadata[ 'id' ] - if verbose: - print '# Checking for functional tests in changeset revision %s of %s, tool ID %s.' % \ - ( changeset_revision, name, tool_id ) + tool_version = tool_metadata[ 'version' ] + tool_guid = tool_metadata[ 'guid' ] + if verbosity >= 2: + print "# Checking tool ID '%s' in changeset revision %s of %s." % \ + ( tool_id, changeset_revision, name ) # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: - if verbose: + if verbosity >= 2: print '# No functional tests defined for %s.' % tool_id # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", # "python_version": "2.7.2", # "architecture": "x86_64", # "system": "Darwin 12.2.0" @@ -172,22 +218,44 @@ # "test_errors": # [ # { - # "test_id": "Something that will easily identify what the problem is", - # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. # }, # ] # } - # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. - test_id = 'Functional tests for %s' % tool_id - test_errors = dict( stderr='No functional tests defined for tool %s in changeset revision %s of repository %s owned by %s.' % \ - ( tool_id, changeset_revision, name, owner ) ) - repository_status[ 'test_errors' ].append( test_errors ) - repository_status[ 'status' ] = 'failed' + # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other + # than the list relevant to what it is testing. + test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid ) + repository_status[ 'invalid_tests' ].append( test_errors ) no_tests += 1 else: + if verbosity >= 2: + print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ + ( tool_id, changeset_revision, name ) has_tests += 1 - if verbose: - if not repository_status[ 'test_errors' ]: + if verbosity >= 1: + if not repository_status[ 'invalid_tests' ]: print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) else: print '# Some tools missing functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) @@ -211,12 +279,17 @@ if os.path.exists( work_dir ): shutil.rmtree( work_dir ) if not has_test_data: - if verbose: + if verbosity >= 1: print '# Test data missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", # "python_version": "2.7.2", # "architecture": "x86_64", # "system": "Darwin 12.2.0" @@ -224,23 +297,44 @@ # "test_errors": # [ # { - # "test_id": "Something that will easily identify what the problem is", - # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. # }, # ] # } - # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. - test_id = 'Find functional test data for %s' % metadata_record.repository.name - test_errors = dict( stderr='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) - repository_status[ 'test_errors' ].append( test_errors ) - repository_status[ 'status' ] = 'failed' + # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other + # than the list relevant to what it is testing. + test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, + reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) + repository_status[ 'invalid_tests' ].append( test_errors ) else: - if verbose: + if verbosity >= 1: print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if not info_only: # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform # on which this script was run. - if repository_status[ 'test_errors' ]: + if repository_status[ 'invalid_tests' ]: # If functional test definitions or test data are missing, set do_not_test = True if and only if: # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. In this case, # the revision will never be updated with correct data, and re-testing it would be redundant. @@ -250,8 +344,6 @@ if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - else: - repository_status[ 'status' ] = 'passed' metadata_record.tool_test_errors = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) diff -r e1e8ddf3401ab1e028fbebd9fa8f4307b8d3ae03 -r c937f80188d8026f2074f8a897e2ff2972d8da6d test/base/util.py --- a/test/base/util.py +++ b/test/base/util.py @@ -74,8 +74,8 @@ return None, repository_name, changeset_revision return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision -def get_test_environment(): - rval = {} +def get_test_environment( current_environment={} ): + rval = current_environment rval[ 'python_version' ] = platform.python_version() rval[ 'architecture' ] = platform.machine() os, hostname, os_version, uname, arch, processor = platform.uname() diff -r e1e8ddf3401ab1e028fbebd9fa8f4307b8d3ae03 -r c937f80188d8026f2074f8a897e2ff2972d8da6d test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -60,6 +60,7 @@ import nose.config import nose.loader import nose.plugins.manager +from nose.plugins import Plugin from base.util import parse_tool_panel_config, get_database_version, get_test_environment, get_repository_current_revision @@ -115,9 +116,36 @@ else: galaxy_encode_secret = os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] + +class ReportResults( Plugin ): + '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' + name = "reportresults" + passed = [] + + def options( self, parser, env=os.environ ): + super( ReportResults, self ).options( parser, env=env ) + + def configure(self, options, conf): + super( ReportResults, self ).configure( options, conf ) + if not self.enabled: + return + + def addSuccess( self, test ): + '''Only record test IDs that correspond to tool functional tests.''' + if 'TestForTool' in test.id(): + test_id = test.id() + # Rearrange the test ID to match the format that is produced in test_results.failures + test_id_parts = test_id.split( '.' ) + fixed_test_id = '%s (%s)' % ( test_id_parts[ -1 ], '.'.join( test_id_parts[ :-1 ] ) ) + self.passed.append( fixed_test_id ) + + def getTestStatus( self ): + return self.passed + def execute_uninstall_method( repository_dict ): # Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them - # and try to re-run the tests after uninstalling the repository. + # and try to re-run the tests after uninstalling the repository, which will cause false failure reports, + # since the test data has been deleted from disk by now. tests_to_delete = [] for key in test_toolbox.__dict__: if key.startswith( 'TestForTool_' ): @@ -132,7 +160,7 @@ test_config.configure( sys.argv ) # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed # repository and delete it from disk. - result = run_tests( test_config ) + result, _ = run_tests( test_config ) success = result.wasSuccessful() return success @@ -191,6 +219,23 @@ else: raise AssertonError( 'Unknown format %s.' % format ) +def get_tool_info_from_test_id( test_id ): + ''' + Test IDs come in the form test_tool_number (functional.test_toolbox.TestForTool_toolshed_url/repos/owner/repository_name/tool_id/tool_version) + We want the tool ID and tool version. + ''' + parts = test_id.replace( ')', '' ).split( '/' ) + tool_version = parts[ -1 ] + tool_id = parts[ -2 ] + return tool_id, tool_version + +def get_tool_test_errors_from_api( tool_shed_url, metadata_dict ): + params = dict() + api_path = metadata_dict[ 'url' ].split( '/' ) + api_url = get_api_url( base=tool_shed_url, parts=api_path ) + repository_metadata = json_from_url( api_url ) + return repository_metadata[ 'tool_test_errors' ] + def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() @@ -201,16 +246,15 @@ if tests_passed: params[ 'tools_functionally_correct' ] = 'true' params[ 'do_not_test' ] = 'true' - test_results_dict[ 'status' ] = 'passed' else: params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'true' - test_results_dict[ 'status' ] = 'failed' params[ 'tool_test_errors' ] = test_results_dict return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) def run_tests( test_config ): loader = nose.loader.TestLoader( config=test_config ) + test_config.plugins.addPlugin( ReportResults() ) plug_loader = test_config.plugins.prepareTestLoader( loader ) if plug_loader is not None: loader = plug_loader @@ -221,7 +265,8 @@ plug_runner = test_config.plugins.prepareTestRunner( test_runner ) if plug_runner is not None: test_runner = plug_runner - return test_runner.run( tests ) + result = test_runner.run( tests ) + return result, test_config.plugins._plugins def main(): # ---- Configuration ------------------------------------------------------ @@ -391,10 +436,6 @@ log.info( "The embedded Galaxy application is running on %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) success = False - repository_status = dict() - test_environment = get_test_environment() - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) try: # Iterate through a list of repository info dicts. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) @@ -472,7 +513,7 @@ test_config.configure( sys.argv ) # Run the configured install method as a test. This method uses the embedded Galaxy application's web interface to install the specified # repository, with tool and repository dependencies also selected for installation. - result = run_tests( test_config ) + result, _ = run_tests( test_config ) success = result.wasSuccessful() # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to # sh run_functional_tests.sh -installed @@ -483,14 +524,63 @@ # and the tools_functionally_correct flag to False, as well as updating tool_test_errors. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) + # The repository_status dict should always have the following structure: + # { + # "test_environment": + # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", + # "python_version": "2.7.2", + # "architecture": "x86_64", + # "system": "Darwin 12.2.0" + # }, + # "test_errors": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. + # }, + # ] + # } + repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, repository_dict ) + test_environment = get_test_environment( repository_status[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'tests_passed' ] = [] + repository_status[ 'test_errors' ] = [] if not has_test_data: log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) repository_status[ 'test_environment' ] = get_test_environment() test_id = 'Find functional test data for %s' % repository_dict[ 'name' ] - test_errors = dict( test_id=test_id, - stdout='No test data found for changeset revision %s of repository %s owned by %s.' % \ - ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ], repository_dict[ 'owner' ] ) ) - repository_status[ 'test_errors' ] = [ test_errors ] + # Record the lack of test data. + test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, + reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) + repository_status[ 'invalid_tests' ].append( test_errors ) # Record the status of this repository in the tool shed. register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -518,12 +608,20 @@ test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) # Run the configured tests. - result = run_tests( test_config ) + result, test_plugins = run_tests( test_config ) success = result.wasSuccessful() # Record some information about the environment in which this test was run, in case a failure is specific to a certain processor # architecture or operating system. repository_dict[ 'test_environment' ] = test_environment - test_errors = [] + for plugin in test_plugins: + if hasattr( plugin, 'getTestStatus' ): + tests_passed = plugin.getTestStatus() + break + repository_status[ 'tests_passed' ] = [] + for test_id in tests_passed: + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + repository_status[ 'tests_passed' ].append( test_result ) if success: # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API @@ -531,15 +629,16 @@ # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) repository_status[ 'test_environment' ] = test_environment - repository_status[ 'test_errors' ] = [] register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. for failure in result.failures: - # Record the twill test identifier, so the repository owner can discover which test is failing. - test_status = dict( test_id=str( failure[0] ) ) + # Record the twill test identifier and information about the tool, so the repository owner can discover which test is failing. + test_id = str( failure[0] ) + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_status = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) log_output = failure[1].replace( '\\n', '\n' ) # Remove debug output that the reviewer or owner doesn't need. log_output = re.sub( r'control \d+:.+', r'', log_output ) @@ -547,7 +646,7 @@ appending_to = 'output' tmp_output = {} output = {} - # Iterate through the functional test output and extract only the important data. Captured logging is not recorded. + # Iterate through the functional test output and extract only the important data. Captured logging and stdout are not recorded. for line in log_output.split( '\n' ): if line.startswith( 'Traceback' ): appending_to = 'traceback' @@ -567,17 +666,15 @@ if appending_to not in tmp_output: tmp_output[ appending_to ] = [] tmp_output[ appending_to ].append( line ) - for output_type in [ 'stderr', 'stdout', 'traceback' ]: + for output_type in [ 'stderr', 'traceback' ]: if output_type in tmp_output: test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - test_errors.append( test_status ) - if test_errors: - # Only update test_errors for this repository if it's not empty. - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'test_errors' ] = test_errors + repository_status[ 'test_errors' ].append( test_status ) + log.debug( to_json_string( repository_status, indent=2, sort_keys=True ) ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome - # status of the tests, and updates tool_test_errors with the relevant log data. - # This also sets the do_not_test and tools_functionally correct flags, and updates the time_last_tested field to today's date. + # of the tests, and updates tool_test_errors with the relevant log data. + # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested + # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Added even more information to the tool_test_errors dict. Separated failed and passed tests in the metadata record. Added documentation.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/b98890b116c0/ changeset: b98890b116c0 user: inithello date: 2013-03-08 20:25:06 summary: Added even more information to the tool_test_errors dict. Separated failed and passed tests in the metadata record. Added documentation. affected #: 3 files diff -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b -r b98890b116c09a286d989b4b41595c0cf10d9c9d lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -38,7 +38,16 @@ '''Script that checks repositories to see if the tools contained within them have functional tests defined.''' parser = OptionParser() parser.add_option( "-i", "--info_only", action="store_true", dest="info_only", help="info about the requested action", default=False ) - parser.add_option( "-v", "--verbose", action="store_true", dest="verbose", help="verbose mode, print the name, owner, and changeset revision of each repository", default=False ) + parser.add_option( + "-v", "--verbose", + action="count", dest="verbosity", + default=1, + help="Control the amount of detail in the log output.") + parser.add_option( + "--verbosity", action="store", dest="verbosity", + metavar='VERBOSITY', + type="int", help="Control the amount of detail in the log output. --verbosity=1 is " + "the same as -v") ( options, args ) = parser.parse_args() ini_file = args[0] config_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) @@ -56,12 +65,12 @@ if options.info_only: print "# Displaying info only ( --info_only )" - if options.verbose: - print "# Displaying extra information ( --verbose )" + if options.verbosity: + print "# Displaying extra information ( --verbosity = %d )" % options.verbosity - check_and_flag_repositories( app, info_only=options.info_only, verbose=options.verbose ) + check_and_flag_repositories( app, info_only=options.info_only, verbosity=options.verbosity ) -def check_and_flag_repositories( app, info_only=False, verbose=False ): +def check_and_flag_repositories( app, info_only=False, verbosity=1 ): ''' This method will iterate through all records in the repository_metadata table, checking each one for tool metadata, then checking the tool metadata for tests. @@ -97,10 +106,15 @@ and test repositories script to process. If the tested changeset revision does not have a test-data directory, this script will also mark the revision not to be tested. - If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute with the following structure: + If any error is encountered, the script will update the repository_metadata.tool_test_errors attribute following this structure: { "test_environment": { + "galaxy_revision": "9001:abcd1234", + "galaxy_database_version": "114", + "tool_shed_revision": "9001:abcd1234", + "tool_shed_mercurial_version": "2.3.1", + "tool_shed_database_version": "17", "python_version": "2.7.2", "architecture": "x86_64", "system": "Darwin 12.2.0" @@ -108,8 +122,28 @@ "test_errors": [ { - "test_id": "Something that will easily identify what the problem is", - "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + "test_id": "The test ID, generated by twill", + "tool_id": "The tool ID that was tested", + "tool_version": "The tool version that was tested", + "stderr": "The output of the test, or a more detailed description of what was tested and what the error was." + "traceback": "The traceback, if any." + }, + ] + "passed_tests": + [ + { + "test_id": "The test ID, generated by twill", + "tool_id": "The tool ID that was tested", + "tool_version": "The tool version that was tested", + }, + ] + "invalid_tests": + [ + { + "tool_id": "The tool ID that does not have functional tests defined.", + "tool_version": "The version of the tool." + "tool_guid": "The guid of the tool." + "reason_test_is_invalid": "A short explanation of what is invalid. }, ] } @@ -129,19 +163,24 @@ repository_status[ 'test_environment' ][ 'tool_shed_database_version' ] = get_database_version( app ) repository_status[ 'test_environment' ][ 'tool_shed_mercurial_version' ] = __version__.version repository_status[ 'test_environment' ][ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) - repository_status[ 'test_errors' ] = [] + repository_status[ 'invalid_tests' ] = [] metadata_records_to_check = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, app.model.RepositoryMetadata.table.c.do_not_test == False, app.model.RepositoryMetadata.table.c.tools_functionally_correct == False ) ) \ .all() for metadata_record in metadata_records_to_check: - repository_status[ 'test_errors' ] = [] + if metadata_record.tool_test_errors: + repository_status = metadata_record.tool_test_errors name = metadata_record.repository.name owner = metadata_record.repository.user.username changeset_revision = str( metadata_record.changeset_revision ) + repository_status[ 'invalid_tests' ] = [] if metadata_record.repository.id not in checked_repository_ids: checked_repository_ids.append( metadata_record.repository.id ) + if verbosity >= 1: + print '# Checking for functional tests in changeset revision %s of %s, owned by %s.' % \ + ( changeset_revision, name, owner ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: @@ -152,19 +191,26 @@ for tool_metadata in metadata_record.metadata[ 'tools' ]: tool_count += 1 tool_id = tool_metadata[ 'id' ] - if verbose: - print '# Checking for functional tests in changeset revision %s of %s, tool ID %s.' % \ - ( changeset_revision, name, tool_id ) + tool_version = tool_metadata[ 'version' ] + tool_guid = tool_metadata[ 'guid' ] + if verbosity >= 2: + print "# Checking tool ID '%s' in changeset revision %s of %s." % \ + ( tool_id, changeset_revision, name ) # If there are no tests, this tool should not be tested, since the tool functional tests only report failure if the test itself fails, # not if it's missing or undefined. Filtering out those repositories at this step will reduce the number of "false negatives" the # automated functional test framework produces. if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: - if verbose: + if verbosity >= 2: print '# No functional tests defined for %s.' % tool_id # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", # "python_version": "2.7.2", # "architecture": "x86_64", # "system": "Darwin 12.2.0" @@ -172,22 +218,44 @@ # "test_errors": # [ # { - # "test_id": "Something that will easily identify what the problem is", - # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. # }, # ] # } - # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. - test_id = 'Functional tests for %s' % tool_id - test_errors = dict( stderr='No functional tests defined for tool %s in changeset revision %s of repository %s owned by %s.' % \ - ( tool_id, changeset_revision, name, owner ) ) - repository_status[ 'test_errors' ].append( test_errors ) - repository_status[ 'status' ] = 'failed' + # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other + # than the list relevant to what it is testing. + test_errors = dict( tool_id=tool_id, tool_version=tool_version, tool_guid=tool_guid ) + repository_status[ 'invalid_tests' ].append( test_errors ) no_tests += 1 else: + if verbosity >= 2: + print "# Tool ID '%s' in changeset revision %s of %s has one or more valid functional tests defined." % \ + ( tool_id, changeset_revision, name ) has_tests += 1 - if verbose: - if not repository_status[ 'test_errors' ]: + if verbosity >= 1: + if not repository_status[ 'invalid_tests' ]: print '# All tools have functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) else: print '# Some tools missing functional tests in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) @@ -211,12 +279,17 @@ if os.path.exists( work_dir ): shutil.rmtree( work_dir ) if not has_test_data: - if verbose: + if verbosity >= 1: print '# Test data missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", # "python_version": "2.7.2", # "architecture": "x86_64", # "system": "Darwin 12.2.0" @@ -224,23 +297,44 @@ # "test_errors": # [ # { - # "test_id": "Something that will easily identify what the problem is", - # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. # }, # ] # } - # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. - test_id = 'Find functional test data for %s' % metadata_record.repository.name - test_errors = dict( stderr='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) - repository_status[ 'test_errors' ].append( test_errors ) - repository_status[ 'status' ] = 'failed' + # Optionally, "traceback" may be included in a test_errors dict, if it is relevant. No script should overwrite anything other + # than the list relevant to what it is testing. + test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, + reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) + repository_status[ 'invalid_tests' ].append( test_errors ) else: - if verbose: + if verbosity >= 1: print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if not info_only: # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform # on which this script was run. - if repository_status[ 'test_errors' ]: + if repository_status[ 'invalid_tests' ]: # If functional test definitions or test data are missing, set do_not_test = True if and only if: # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. In this case, # the revision will never be updated with correct data, and re-testing it would be redundant. @@ -250,8 +344,6 @@ if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - else: - repository_status[ 'status' ] = 'passed' metadata_record.tool_test_errors = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) diff -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b -r b98890b116c09a286d989b4b41595c0cf10d9c9d test/base/util.py --- a/test/base/util.py +++ b/test/base/util.py @@ -74,8 +74,8 @@ return None, repository_name, changeset_revision return last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision -def get_test_environment(): - rval = {} +def get_test_environment( current_environment={} ): + rval = current_environment rval[ 'python_version' ] = platform.python_version() rval[ 'architecture' ] = platform.machine() os, hostname, os_version, uname, arch, processor = platform.uname() diff -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b -r b98890b116c09a286d989b4b41595c0cf10d9c9d test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -60,6 +60,7 @@ import nose.config import nose.loader import nose.plugins.manager +from nose.plugins import Plugin from base.util import parse_tool_panel_config, get_database_version, get_test_environment, get_repository_current_revision @@ -115,9 +116,36 @@ else: galaxy_encode_secret = os.environ[ 'GALAXY_INSTALL_TEST_SECRET' ] + +class ReportResults( Plugin ): + '''Simple Nose plugin to record the IDs of all tests run, regardless of success.''' + name = "reportresults" + passed = [] + + def options( self, parser, env=os.environ ): + super( ReportResults, self ).options( parser, env=env ) + + def configure(self, options, conf): + super( ReportResults, self ).configure( options, conf ) + if not self.enabled: + return + + def addSuccess( self, test ): + '''Only record test IDs that correspond to tool functional tests.''' + if 'TestForTool' in test.id(): + test_id = test.id() + # Rearrange the test ID to match the format that is produced in test_results.failures + test_id_parts = test_id.split( '.' ) + fixed_test_id = '%s (%s)' % ( test_id_parts[ -1 ], '.'.join( test_id_parts[ :-1 ] ) ) + self.passed.append( fixed_test_id ) + + def getTestStatus( self ): + return self.passed + def execute_uninstall_method( repository_dict ): # Delete any configured tool functional tests from the test_toolbox.__dict__, otherwise nose will find them - # and try to re-run the tests after uninstalling the repository. + # and try to re-run the tests after uninstalling the repository, which will cause false failure reports, + # since the test data has been deleted from disk by now. tests_to_delete = [] for key in test_toolbox.__dict__: if key.startswith( 'TestForTool_' ): @@ -132,7 +160,7 @@ test_config.configure( sys.argv ) # Run the uninstall method. This method uses the Galaxy web interface to uninstall the previously installed # repository and delete it from disk. - result = run_tests( test_config ) + result, _ = run_tests( test_config ) success = result.wasSuccessful() return success @@ -191,6 +219,23 @@ else: raise AssertonError( 'Unknown format %s.' % format ) +def get_tool_info_from_test_id( test_id ): + ''' + Test IDs come in the form test_tool_number (functional.test_toolbox.TestForTool_toolshed_url/repos/owner/repository_name/tool_id/tool_version) + We want the tool ID and tool version. + ''' + parts = test_id.replace( ')', '' ).split( '/' ) + tool_version = parts[ -1 ] + tool_id = parts[ -2 ] + return tool_id, tool_version + +def get_tool_test_errors_from_api( tool_shed_url, metadata_dict ): + params = dict() + api_path = metadata_dict[ 'url' ].split( '/' ) + api_url = get_api_url( base=tool_shed_url, parts=api_path ) + repository_metadata = json_from_url( api_url ) + return repository_metadata[ 'tool_test_errors' ] + def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() @@ -201,16 +246,15 @@ if tests_passed: params[ 'tools_functionally_correct' ] = 'true' params[ 'do_not_test' ] = 'true' - test_results_dict[ 'status' ] = 'passed' else: params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'true' - test_results_dict[ 'status' ] = 'failed' params[ 'tool_test_errors' ] = test_results_dict return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) def run_tests( test_config ): loader = nose.loader.TestLoader( config=test_config ) + test_config.plugins.addPlugin( ReportResults() ) plug_loader = test_config.plugins.prepareTestLoader( loader ) if plug_loader is not None: loader = plug_loader @@ -221,7 +265,8 @@ plug_runner = test_config.plugins.prepareTestRunner( test_runner ) if plug_runner is not None: test_runner = plug_runner - return test_runner.run( tests ) + result = test_runner.run( tests ) + return result, test_config.plugins._plugins def main(): # ---- Configuration ------------------------------------------------------ @@ -391,10 +436,6 @@ log.info( "The embedded Galaxy application is running on %s:%s" % ( galaxy_test_host, galaxy_test_port ) ) log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) success = False - repository_status = dict() - test_environment = get_test_environment() - test_environment[ 'galaxy_database_version' ] = get_database_version( app ) - test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) try: # Iterate through a list of repository info dicts. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) @@ -472,7 +513,7 @@ test_config.configure( sys.argv ) # Run the configured install method as a test. This method uses the embedded Galaxy application's web interface to install the specified # repository, with tool and repository dependencies also selected for installation. - result = run_tests( test_config ) + result, _ = run_tests( test_config ) success = result.wasSuccessful() # If the installation succeeds, configure and run functional tests for this repository. This is equivalent to # sh run_functional_tests.sh -installed @@ -483,14 +524,63 @@ # and the tools_functionally_correct flag to False, as well as updating tool_test_errors. file( galaxy_shed_tools_dict, 'w' ).write( to_json_string( dict() ) ) has_test_data, shed_tools_dict = parse_tool_panel_config( galaxy_shed_tool_conf_file, from_json_string( file( galaxy_shed_tools_dict, 'r' ).read() ) ) + # The repository_status dict should always have the following structure: + # { + # "test_environment": + # { + # "galaxy_revision": "9001:abcd1234", + # "galaxy_database_version": "114", + # "tool_shed_revision": "9001:abcd1234", + # "tool_shed_mercurial_version": "2.3.1", + # "tool_shed_database_version": "17", + # "python_version": "2.7.2", + # "architecture": "x86_64", + # "system": "Darwin 12.2.0" + # }, + # "test_errors": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # "traceback": "The captured traceback." + # }, + # ] + # "passed_tests": + # [ + # { + # "test_id": "The test ID, generated by twill", + # "tool_id": "The tool ID that was tested", + # "tool_version": "The tool version that was tested", + # "stderr": "The output of the test, or a more detailed description of what was tested and what the outcome was." + # }, + # ] + # "invalid_tests": + # [ + # { + # "tool_id": "The tool ID that does not have functional tests defined.", + # "tool_version": "The version of the tool." + # "tool_guid": "The guid of the tool." + # "reason_test_is_invalid": "A short explanation of what is invalid. + # }, + # ] + # } + repository_status = get_tool_test_errors_from_api( galaxy_tool_shed_url, repository_dict ) + test_environment = get_test_environment( repository_status[ 'test_environment' ] ) + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'tests_passed' ] = [] + repository_status[ 'test_errors' ] = [] if not has_test_data: log.error( 'Test data is missing for this repository. Updating repository and skipping functional tests.' ) repository_status[ 'test_environment' ] = get_test_environment() test_id = 'Find functional test data for %s' % repository_dict[ 'name' ] - test_errors = dict( test_id=test_id, - stdout='No test data found for changeset revision %s of repository %s owned by %s.' % \ - ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ], repository_dict[ 'owner' ] ) ) - repository_status[ 'test_errors' ] = [ test_errors ] + # Record the lack of test data. + test_errors = dict( tool_id=None, tool_version=None, tool_guid=None, + reason_test_is_invalid="Repository %s is missing a test-data directory." % name ) + repository_status[ 'invalid_tests' ].append( test_errors ) # Record the status of this repository in the tool shed. register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -518,12 +608,20 @@ test_config = nose.config.Config( env=os.environ, plugins=nose.plugins.manager.DefaultPluginManager() ) test_config.configure( sys.argv ) # Run the configured tests. - result = run_tests( test_config ) + result, test_plugins = run_tests( test_config ) success = result.wasSuccessful() # Record some information about the environment in which this test was run, in case a failure is specific to a certain processor # architecture or operating system. repository_dict[ 'test_environment' ] = test_environment - test_errors = [] + for plugin in test_plugins: + if hasattr( plugin, 'getTestStatus' ): + tests_passed = plugin.getTestStatus() + break + repository_status[ 'tests_passed' ] = [] + for test_id in tests_passed: + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_result = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) + repository_status[ 'tests_passed' ].append( test_result ) if success: # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API @@ -531,15 +629,16 @@ # updates the time_last_tested field to today's date. repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) repository_status[ 'test_environment' ] = test_environment - repository_status[ 'test_errors' ] = [] register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) else: # If the functional tests fail, log the output and update the failed changeset revision's metadata record in the tool shed via the API. for failure in result.failures: - # Record the twill test identifier, so the repository owner can discover which test is failing. - test_status = dict( test_id=str( failure[0] ) ) + # Record the twill test identifier and information about the tool, so the repository owner can discover which test is failing. + test_id = str( failure[0] ) + tool_id, tool_version = get_tool_info_from_test_id( test_id ) + test_status = dict( test_id=test_id, tool_id=tool_id, tool_version=tool_version ) log_output = failure[1].replace( '\\n', '\n' ) # Remove debug output that the reviewer or owner doesn't need. log_output = re.sub( r'control \d+:.+', r'', log_output ) @@ -547,7 +646,7 @@ appending_to = 'output' tmp_output = {} output = {} - # Iterate through the functional test output and extract only the important data. Captured logging is not recorded. + # Iterate through the functional test output and extract only the important data. Captured logging and stdout are not recorded. for line in log_output.split( '\n' ): if line.startswith( 'Traceback' ): appending_to = 'traceback' @@ -567,17 +666,15 @@ if appending_to not in tmp_output: tmp_output[ appending_to ] = [] tmp_output[ appending_to ].append( line ) - for output_type in [ 'stderr', 'stdout', 'traceback' ]: + for output_type in [ 'stderr', 'traceback' ]: if output_type in tmp_output: test_status[ output_type ] = '\n'.join( tmp_output[ output_type ] ) - test_errors.append( test_status ) - if test_errors: - # Only update test_errors for this repository if it's not empty. - repository_status[ 'test_environment' ] = test_environment - repository_status[ 'test_errors' ] = test_errors + repository_status[ 'test_errors' ].append( test_status ) + log.debug( to_json_string( repository_status, indent=2, sort_keys=True ) ) # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome - # status of the tests, and updates tool_test_errors with the relevant log data. - # This also sets the do_not_test and tools_functionally correct flags, and updates the time_last_tested field to today's date. + # of the tests, and updates tool_test_errors with the relevant log data. + # This also sets the do_not_test and tools_functionally correct flags to the appropriate values, and updates the time_last_tested + # field to today's date. repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a2cfc332d7b8/ changeset: a2cfc332d7b8 user: Kyle Ellrott date: 2013-01-30 00:58:58 summary: Enabling dataset UUID setting through library upload, upload1 tool, and galaxy.json file affected #: 5 files diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 lib/galaxy/jobs/__init__.py --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -907,7 +907,6 @@ job_context = ExpressionContext( dict( stdout = job.stdout, stderr = job.stderr ) ) job_tool = self.app.toolbox.tools_by_id.get( job.tool_id, None ) - for dataset_assoc in job.output_datasets + job.output_library_datasets: context = self.get_dataset_finish_context( job_context, dataset_assoc.dataset.dataset ) #should this also be checking library associations? - can a library item be added from a history before the job has ended? - lets not allow this to occur @@ -917,6 +916,8 @@ dataset.info = ( dataset.info or '' ) + context['stdout'] + context['stderr'] dataset.tool_version = self.version_string dataset.set_size() + if 'uuid' in context: + dataset.dataset.uuid = context['uuid'] # Update (non-library) job output datasets through the object store if dataset not in job.output_library_datasets: self.app.object_store.update_from_file(dataset.dataset, create=True) diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 lib/galaxy/tools/actions/upload_common.py --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -294,6 +294,10 @@ link_data_only = uploaded_dataset.link_data_only except: link_data_only = 'copy_files' + try: + uuid_str = uploaded_dataset.uuid + except: + uuid_str = None json = dict( file_type = uploaded_dataset.file_type, ext = uploaded_dataset.ext, name = uploaded_dataset.name, @@ -302,6 +306,7 @@ type = uploaded_dataset.type, is_binary = is_binary, link_data_only = link_data_only, + uuid = uuid_str, space_to_tab = uploaded_dataset.space_to_tab, in_place = trans.app.config.external_chown_script is None, path = uploaded_dataset.path ) diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 lib/galaxy/webapps/galaxy/controllers/library_common.py --- a/lib/galaxy/webapps/galaxy/controllers/library_common.py +++ b/lib/galaxy/webapps/galaxy/controllers/library_common.py @@ -1070,11 +1070,13 @@ job, output = upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder ) # HACK: Prevent outputs_to_working_directory from overwriting inputs when "linking" job.add_parameter( 'link_data_only', to_json_string( kwd.get( 'link_data_only', 'copy_files' ) ) ) + job.add_parameter( 'uuid', to_json_string( kwd.get( 'uuid', None ) ) ) trans.sa_session.add( job ) trans.sa_session.flush() return output def make_library_uploaded_dataset( self, trans, cntrller, params, name, path, type, library_bunch, in_folder=None ): link_data_only = params.get( 'link_data_only', 'copy_files' ) + uuid_str = params.get( 'uuid', None ) library_bunch.replace_dataset = None # not valid for these types of upload uploaded_dataset = util.bunch.Bunch() new_name = name @@ -1096,6 +1098,7 @@ uploaded_dataset.in_folder = in_folder uploaded_dataset.data = upload_common.new_upload( trans, cntrller, uploaded_dataset, library_bunch ) uploaded_dataset.link_data_only = link_data_only + uploaded_dataset.uuid = uuid_str if link_data_only == 'link_to_files': uploaded_dataset.data.file_name = os.path.abspath( path ) # Since we are not copying the file into Galaxy's managed diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 scripts/api/load_data_with_metadata.py --- a/scripts/api/load_data_with_metadata.py +++ b/scripts/api/load_data_with_metadata.py @@ -15,10 +15,34 @@ import sys import json import time +import argparse + sys.path.insert( 0, os.path.dirname( __file__ ) ) from common import submit, display -def main(api_key, api_url, in_folder, data_library): +def load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field=None): + data = {} + data['folder_id'] = library_folder_id + data['file_type'] = 'auto' + data['dbkey'] = '' + data['upload_option'] = 'upload_paths' + data['filesystem_paths'] = fullpath + data['create_type'] = 'file' + data['link_data_only'] = 'link_to_files' + + handle = open( fullpath + ".json" ) + smeta = handle.read() + handle.close() + ext_meta = json.loads(smeta) + data['extended_metadata'] = ext_meta + if uuid_field is not None and uuid_field in ext_meta: + data['uuid'] = ext_meta[uuid_field] + + libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted = True) + print libset + + +def main(api_key, api_url, in_folder, data_library, uuid_field=None): # Find/Create data library with the above name. Assume we're putting datasets in the root folder '/' libs = display(api_key, api_url + 'libraries', return_formatted=False) library_id = None @@ -28,7 +52,7 @@ if not library_id: lib_create_data = {'name':data_library} library = submit(api_key, api_url + 'libraries', lib_create_data, return_formatted=False) - library_id = library[0]['id'] + library_id = library['id'] folders = display(api_key, api_url + "libraries/%s/contents" % library_id, return_formatted = False) for f in folders: if f['name'] == "/": @@ -37,35 +61,24 @@ print "Failure to configure library destination." sys.exit(1) - for fname in os.listdir(in_folder): - fullpath = os.path.join(in_folder, fname) - if os.path.isfile(fullpath) and os.path.exists(fullpath + ".json"): + if os.path.isfile(in_folder): + if os.path.exists(in_folder + ".json"): + fullpath = os.path.abspath(in_folder) print "Loading", fullpath - data = {} - data['folder_id'] = library_folder_id - data['file_type'] = 'auto' - data['dbkey'] = '' - data['upload_option'] = 'upload_paths' - data['filesystem_paths'] = fullpath - data['create_type'] = 'file' - - data['link_data_only'] = 'link_to_files' - - handle = open( fullpath + ".json" ) - smeta = handle.read() - handle.close() - data['extended_metadata'] = json.loads(smeta) - libset = submit(api_key, api_url + "libraries/%s/contents" % library_id, data, return_formatted = True) - print libset + load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field) + else: + for fname in os.listdir(in_folder): + fullpath = os.path.join(in_folder, fname) + if os.path.isfile(fullpath) and os.path.exists(fullpath + ".json"): + print "Loading", fullpath + load_file(fullpath, api_key, api_url, library_id, library_folder_id, uuid_field) if __name__ == '__main__': - try: - api_key = sys.argv[1] - api_url = sys.argv[2] - in_folder = sys.argv[3] - data_library = sys.argv[4] - except IndexError: - print 'usage: %s key url in_folder data_library' % os.path.basename( sys.argv[0] ) - sys.exit( 1 ) - main(api_key, api_url, in_folder, data_library ) - + parser = argparse.ArgumentParser() + parser.add_argument("api_key", help="API KEY") + parser.add_argument('api_url', help='API URL') + parser.add_argument("in_folder", help="Input Folder") + parser.add_argument("data_library", help="Data Library") + parser.add_argument("--uuid_field", help="UUID Field", default=None) + args = parser.parse_args() + main(args.api_key, args.api_url, args.in_folder, args.data_library, args.uuid_field) diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 tools/data_source/upload.py --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -307,6 +307,8 @@ stdout = stdout, name = dataset.name, line_count = line_count ) + if dataset.get('uuid', None) is not None: + info['uuid'] = dataset.get('uuid') json_file.write( to_json_string( info ) + "\n" ) if link_data_only == 'copy_files' and datatype.dataset_content_needs_grooming( output_path ): https://bitbucket.org/galaxy/galaxy-central/commits/ddcce8b55152/ changeset: ddcce8b55152 user: dannon date: 2013-03-08 17:35:06 summary: Merge changes from PR https://bitbucket.org/galaxy/galaxy-central/pull-request/115/adding-dataset… affected #: 6 files diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f lib/galaxy/model/__init__.py --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1774,6 +1774,10 @@ misc_info = ldda.info, misc_blurb = ldda.blurb, template_data = template_data ) + if ldda.dataset.uuid is None: + rval['uuid'] = None + else: + rval['uuid'] = str(ldda.dataset.uuid) for name, spec in ldda.metadata.spec.items(): val = ldda.metadata.get( name ) if isinstance( val, MetadataFile ): diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1922,6 +1922,8 @@ if "runtool_btn" not in incoming and "URL" not in incoming: if not self.display_interface: return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) + if len(incoming): + self.update_state( trans, self.inputs_by_page[state.page], state.inputs, incoming, old_errors=old_errors or {} ) return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -220,6 +220,11 @@ hda_dict[ 'display_apps' ] = get_display_apps( trans, hda ) hda_dict[ 'display_types' ] = get_old_display_applications( trans, hda ) + if hda.dataset.uuid is None: + hda_dict['uuid'] = None + else: + hda_dict['uuid'] = str(hda.dataset.uuid) + hda_dict[ 'visualizations' ] = hda.get_visualizations() if hda.peek and hda.peek != 'no peek': hda_dict[ 'peek' ] = to_unicode( hda.display_peek() ) diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -22,7 +22,8 @@ return trans.fill_template( "root/index.mako", tool_id=tool_id, workflow_id=workflow_id, - m_c=m_c, m_a=m_a ) + m_c=m_c, m_a=m_a, + params=kwd ) ## ---- Tool related ----------------------------------------------------- diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f templates/webapps/galaxy/library/common/ldda_info.mako --- a/templates/webapps/galaxy/library/common/ldda_info.mako +++ b/templates/webapps/galaxy/library/common/ldda_info.mako @@ -100,6 +100,13 @@ ${ldda.get_size( nice_size=True )} <div style="clear: both"></div></div> + %if ldda.dataset.uuid: + <div class="form-row"> + <label>UUID:</label> + ${ldda.dataset.uuid} + <div style="clear: both"></div> + </div> + %endif %if ldda.tool_version: <div class="form-row"><label>Tool version:</label> diff -r a2cfc332d7b8e0f3b765b5d5b3c5b012521c39c0 -r ddcce8b55152c47177d85ab1a2433f80f162b64f templates/webapps/galaxy/root/index.mako --- a/templates/webapps/galaxy/root/index.mako +++ b/templates/webapps/galaxy/root/index.mako @@ -118,7 +118,7 @@ if trans.app.config.require_login and not trans.user: center_url = h.url_for( controller='user', action='login' ) elif tool_id is not None: - center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True ) + center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True, **params ) elif workflow_id is not None: center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) elif m_c is not None: https://bitbucket.org/galaxy/galaxy-central/commits/a37fbe9cbd93/ changeset: a37fbe9cbd93 user: dannon date: 2013-03-08 20:15:11 summary: Merge affected #: 6 files diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -2,15 +2,20 @@ Upload class """ -from galaxy.web.base.controller import * +import logging +import galaxy.util + +from galaxy import web +from galaxy.tools import DefaultToolState +from galaxy.tools.actions import upload_common +from galaxy.tools.parameters import params_to_incoming, visit_input_values +from galaxy.tools.parameters.basic import DataToolParameter, UnvalidatedValue from galaxy.util.bunch import Bunch from galaxy.util.hash_util import is_hashable -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.basic import UnvalidatedValue -from galaxy.tools.parameters import params_to_incoming -from galaxy.tools.actions import upload_common +from galaxy.web import error, url_for +from galaxy.web.base.controller import BaseUIController +from galaxy.web.form_builder import SelectField -import logging log = logging.getLogger( __name__ ) class AddFrameData: @@ -47,7 +52,7 @@ tools = [] tool = None # Backwards compatibility for datasource tools that have default tool_id configured, but which are now using only GALAXY_URL. - tool_ids = util.listify( tool_id ) + tool_ids = galaxy.util.listify( tool_id ) for tool_id in tool_ids: if get_loaded_tools_by_lineage: tools = toolbox.get_loaded_tools_by_lineage( tool_id ) @@ -83,7 +88,7 @@ message=message, status=status, redirect=redirect ) ) - params = util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers + params = galaxy.util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers #do param translation here, used by datasource tools if tool.input_translator: tool.input_translator.translate( params ) @@ -103,7 +108,7 @@ toolbox=self.get_toolbox(), tool_version_select_field=tool_version_select_field, tool=tool, - util=util, + util=galaxy.util, add_frame=add_frame, **vars ) @@ -215,7 +220,7 @@ #create an incoming object from the original job's dataset-modified param objects incoming = {} params_to_incoming( incoming, tool.inputs, params_objects, trans.app ) - incoming[ "tool_state" ] = util.object_to_string( state.encode( tool, trans.app ) ) + incoming[ "tool_state" ] = galaxy.util.object_to_string( state.encode( tool, trans.app ) ) template, vars = tool.handle_input( trans, incoming, old_errors=upgrade_messages ) #update new state with old parameters # Is the "add frame" stuff neccesary here? add_frame = AddFrameData() @@ -228,7 +233,7 @@ toolbox=self.get_toolbox(), tool_version_select_field=tool_version_select_field, tool=tool, - util=util, + util=galaxy.util, add_frame=add_frame, tool_id_version_message=tool_id_version_message, **vars ) @@ -288,9 +293,9 @@ tool = self.get_toolbox().get_tool( tool_id ) if not tool: return False # bad tool_id - nonfile_params = util.Params( kwd, sanitize=False ) + nonfile_params = galaxy.util.Params( kwd, sanitize=False ) if kwd.get( 'tool_state', None ) not in ( None, 'None' ): - encoded_state = util.string_to_object( kwd["tool_state"] ) + encoded_state = galaxy.util.string_to_object( kwd["tool_state"] ) tool_state = DefaultToolState() tool_state.decode( encoded_state, tool, trans.app ) else: diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b test-data/sort_in1.bed --- a/test-data/sort_in1.bed +++ b/test-data/sort_in1.bed @@ -1,3 +1,6 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b test-data/sort_out1.bed --- a/test-data/sort_out1.bed +++ b/test-data/sort_out1.bed @@ -1,29 +1,32 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters +chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, +chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, +chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, +chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, +chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, +chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, +chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, +chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, +chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, +chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, +chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, +chr6 108722790 108950942 NM_145315 0 + 108722976 108950321 0 13 325,224,52,102,131,100,59,83,71,101,141,114,750, 0,28931,52094,60760,61796,71339,107102,152319,181970,182297,215317,224802,227402, +chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, +chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, -chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, -chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, -chr6 108722790 108950942 NM_145315 0 + 108722976 108950321 0 13 325,224,52,102,131,100,59,83,71,101,141,114,750, 0,28931,52094,60760,61796,71339,107102,152319,181970,182297,215317,224802,227402, +chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, +chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, +chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, chr7 113320332 113924911 AK131266 0 + 113862563 113893433 0 20 285,91,178,90,58,75,138,51,201,178,214,105,88,84,77,102,122,70,164,1124, 0,201692,340175,448290,451999,484480,542213,543265,543478,545201,556083,558358,565876,567599,573029,573245,575738,577123,577946,603455, -chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, -chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, -chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, -chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, -chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, chr8 118880786 119193239 NM_000127 0 - 118881131 119192466 0 11 531,172,161,90,96,119,133,120,108,94,1735, 0,5355,7850,13505,19068,20309,23098,30863,36077,37741,310718, +chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, +chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, +chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, +chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, +chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, +chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, +chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, -chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, -chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, -chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, -chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, -chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, -chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, -chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, -chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, -chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, -chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, -chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, -chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, -chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, -chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, -chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, -chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, -chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b test-data/sort_out2.bed --- a/test-data/sort_out2.bed +++ b/test-data/sort_out2.bed @@ -1,3 +1,18 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters +chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, +chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, +chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, +chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, +chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, +chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, +chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, +chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, +chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, +chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, +chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, @@ -6,24 +21,12 @@ chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, -chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, -chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, chr8 118880786 119193239 NM_000127 0 - 118881131 119192466 0 11 531,172,161,90,96,119,133,120,108,94,1735, 0,5355,7850,13505,19068,20309,23098,30863,36077,37741,310718, -chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, -chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, -chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, -chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, -chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, -chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, -chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, -chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, -chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, -chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, -chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b tools/filters/sorter.py --- a/tools/filters/sorter.py +++ b/tools/filters/sorter.py @@ -1,49 +1,59 @@ """ -Sorts tabular data on one or more columns. + Sorts tabular data on one or more columns. All comments of the file are collected + and placed at the beginning of the sorted output file. + + usage: sorter.py [options] + -i, --input: Tabular file to be sorted + -o, --output: Sorted output file + -k, --key: Key (see manual for bash/sort) + + usage: sorter.py input output [key ...] +""" +# 03/05/2013 guerler -usage: %prog [options] - -i, --input=i: Tabular file to be sorted - -o, --out_file1=o: Sorted output file - -c, --column=c: First column to sort on - -s, --style=s: Sort style (numerical or alphabetical) - -r, --order=r: Order (ASC or DESC) +# imports +import os, re, string, sys +from optparse import OptionParser -usage: %prog input out_file1 column style order [column style ...] -""" - -import os, re, string, sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from bx.cookbook import doc_optparse - +# error def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() +# main def main(): - #Parse Command Line - options, args = doc_optparse.parse( __doc__ ) + # define options + parser = OptionParser() + parser.add_option("-i", "--input") + parser.add_option("-o", "--output") + parser.add_option("-k", "--key", action="append") + + # parse + options, args = parser.parse_args() + try: - inputfile = options.input - outputfile = '-o %s' % options.out_file1 - columns = [options.column] - styles = [('','n')[options.style == 'num']] - orders = [('','r')[options.order == 'DESC']] - col_style_orders = sys.argv[6:] - if len(col_style_orders) > 1: - columns.extend([col_style_orders[i] for i in range(0,len(col_style_orders),3)]) - styles.extend([('','n')[col_style_orders[i] == 'num'] for i in range(1,len(col_style_orders),3)]) - orders.extend([('','r')[col_style_orders[i] == 'DESC'] for i in range(2,len(col_style_orders),3)]) - cols = [ '-k%s,%s%s%s'%(columns[i], columns[i], styles[i], orders[i]) for i in range(len(columns)) ] + # retrieve options + input = options.input + output = options.output + key = [" -k" + k for k in options.key] + + # grep comments + grep_comments = "(grep '^#' %s) > %s" % (input, output) + #print grep_comments + + # grep and sort columns + sort_columns = "(grep '^[^#]' %s | sort -f -t '\t' %s) >> %s" % (input, ' '.join(key), output) + #print sort_columns + + # execute + os.system(grep_comments) + os.system(sort_columns) + except Exception, ex: - stop_err('Error parsing input parameters\n' + str(ex)) + stop_err('Error running sorter.py\n' + str(ex)) - # Launch sort. - cmd = "sort -f -t ' ' %s %s %s" % (' '.join(cols), outputfile, inputfile) - try: - os.system(cmd) - except Exception, ex: - stop_err('Error running sort command\n' + str(ex)) + # exit + sys.exit(0) if __name__ == "__main__": main() diff -r ddcce8b55152c47177d85ab1a2433f80f162b64f -r a37fbe9cbd93a4e73a765d50be3aca4f31968f0b tools/filters/sorter.xml --- a/tools/filters/sorter.xml +++ b/tools/filters/sorter.xml @@ -1,130 +1,61 @@ -<tool id="sort1" name="Sort" version="1.0.1"> - <description>data in ascending or descending order</description> - <command interpreter="python"> - sorter.py - --input=$input - --out_file1=$out_file1 - --column=$column - --style=$style - --order=$order - #for $col in $column_set: - ${col.other_column} - ${col.other_style} - ${col.other_order} - #end for - </command> - <inputs> - <param format="tabular" name="input" type="data" label="Sort Dataset" /> - <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/> - <param name="style" type="select" label="with flavor"> - <option value="num">Numerical sort</option> - <option value="alpha">Alphabetical sort</option> - </param> - <param name="order" type="select" label="everything in"> - <option value="DESC">Descending order</option> - <option value="ASC">Ascending order</option> - </param> - <repeat name="column_set" title="Column selection"> - <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /> - <param name="other_style" type="select" label="with flavor"> - <option value="num">Numerical sort</option> - <option value="alpha">Alphabetical sort</option> - </param> - <param name="other_order" type="select" label="everything in"> - <option value="DESC">Descending order</option> - <option value="ASC">Ascending order</option> - </param> - </repeat> - </inputs> - <outputs> - <data format="input" name="out_file1" metadata_source="input"/> - </outputs> - <tests> - <test> - <param name="input" value="sort_in1.bed"/> - <param name="column" value="1"/> - <param name="style" value="num"/> - <param name="order" value="ASC"/> - <param name="other_column" value="3"/> - <param name="other_style" value="num"/> - <param name="other_order" value="ASC"/> - <output name="out_file1" file="sort_out1.bed"/> - </test> - <test> - <param name="input" value="sort_in1.bed"/> - <param name="column" value="3"/> - <param name="style" value="alpha"/> - <param name="order" value="ASC"/> - <param name="other_column" value="1"/> - <param name="other_style" value="alpha"/> - <param name="other_order" value="ASC"/> - <output name="out_file1" file="sort_out2.bed"/> - </test> - </tests> - <help> +<tool id="sort1" name="Sort" version="1.0.2"> + <description>data in ascending or descending order</description> + <command interpreter="python"> + sorter.py + --input=$input + --output=$output + --key=$column,$column$style$order + #for $col in $column_set: + --key=${col.other_column},${col.other_column}${col.other_style}${col.other_order} + #end for + </command> + <inputs> + <param format="tabular" name="input" type="data" label="Sort Dataset" /> + <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/> + <param name="style" type="select" label="with flavor"> + <option value="n">Numerical sort</option> + <option value="">Alphabetical sort</option> + </param> + <param name="order" type="select" label="everything in"> + <option value="r">Descending order</option> + <option value="">Ascending order</option> + </param> + <repeat name="column_set" title="Column selection"> + <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /> + <param name="other_style" type="select" label="with flavor"> + <option value="n">Numerical sort</option> + <option value="">Alphabetical sort</option> + </param> + <param name="other_order" type="select" label="everything in"> + <option value="r">Descending order</option> + <option value="">Ascending order</option> + </param> + </repeat> + </inputs> + <outputs> + <data format="input" name="output" metadata_source="input"/> + </outputs> + <tests> + <test> + <param name="input" value="sort_in1.bed"/> + <param name="column" value="1"/> + <param name="style" value=""/> + <param name="order" value=""/> + <param name="other_column" value="3"/> + <param name="other_style" value="n"/> + <param name="other_order" value="r"/> + <output name="output" file="sort_out1.bed"/> + </test> + <test> + <param name="input" value="sort_in1.bed"/> + <param name="column" value="1"/> + <param name="style" value=""/> + <param name="order" value=""/> + <param name="other_column" value="3"/> + <param name="other_style" value="n"/> + <param name="other_order" value=""/> + <output name="output" file="sort_out2.bed"/> + </test> + </tests> -.. class:: infomark - -**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* - ------ - -**Syntax** - -This tool sorts the dataset on any number of columns in either ascending or descending order. - -* Numerical sort orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. -* Alphabetical sort is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). - ------ - -**Examples** - -The list of numbers 4,17,3,5 collates to 3,4,5,17 by numerical sorting, while it collates to 17,3,4,5 by alphabetical sorting. - -Sorting the following:: - - Q d 7 II jhu 45 - A kk 4 I h 111 - Pd p 1 ktY WS 113 - A g 10 H ZZ 856 - A edf 4 tw b 234 - BBB rt 10 H ZZ 100 - A rew 10 d b 1111 - C sd 19 YH aa 10 - Hah c 23 ver bb 467 - MN gtr 1 a X 32 - N j 9 a T 205 - BBB rrf 10 b Z 134 - odfr ws 6 Weg dew 201 - C f 3 WW SW 34 - A jhg 4 I b 345 - Pd gf 7 Gthe de 567 - rS hty 90 YY LOp 89 - A g 10 H h 43 - A g 4 I h 500 - -on columns 1 (alpha), 3 (num), and 6 (num) in ascending order will yield:: - - A kk 4 I h 111 - A edf 4 tw b 234 - A jhg 4 I b 345 - A g 4 I h 500 - A g 10 H h 43 - A g 10 H ZZ 856 - A rew 10 d b 1111 - BBB rt 10 H ZZ 100 - BBB rrf 10 b Z 134 - C f 3 WW SW 34 - C sd 19 YH aa 10 - Hah c 23 ver bb 467 - MN gtr 1 a X 32 - N j 9 a T 205 - odfr ws 6 Weg dew 201 - Pd p 1 ktY WS 113 - Pd gf 7 Gthe de 567 - Q d 7 II jhu 45 - rS hty 90 YY LOp 89 - - </help></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove import * from tool_runner, fixes rerun (missing visit_input_values function) after recent import refactoring.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/57c80f54cd51/ changeset: 57c80f54cd51 user: dannon date: 2013-03-08 18:29:52 summary: Remove import * from tool_runner, fixes rerun (missing visit_input_values function) after recent import refactoring. affected #: 1 file diff -r 3317f0d7f848445cff6b384bf5a1915db677d139 -r 57c80f54cd51a5b427692df6d29513c88554a112 lib/galaxy/webapps/galaxy/controllers/tool_runner.py --- a/lib/galaxy/webapps/galaxy/controllers/tool_runner.py +++ b/lib/galaxy/webapps/galaxy/controllers/tool_runner.py @@ -2,15 +2,20 @@ Upload class """ -from galaxy.web.base.controller import * +import logging +import galaxy.util + +from galaxy import web +from galaxy.tools import DefaultToolState +from galaxy.tools.actions import upload_common +from galaxy.tools.parameters import params_to_incoming, visit_input_values +from galaxy.tools.parameters.basic import DataToolParameter, UnvalidatedValue from galaxy.util.bunch import Bunch from galaxy.util.hash_util import is_hashable -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.basic import UnvalidatedValue -from galaxy.tools.parameters import params_to_incoming -from galaxy.tools.actions import upload_common +from galaxy.web import error, url_for +from galaxy.web.base.controller import BaseUIController +from galaxy.web.form_builder import SelectField -import logging log = logging.getLogger( __name__ ) class AddFrameData: @@ -47,7 +52,7 @@ tools = [] tool = None # Backwards compatibility for datasource tools that have default tool_id configured, but which are now using only GALAXY_URL. - tool_ids = util.listify( tool_id ) + tool_ids = galaxy.util.listify( tool_id ) for tool_id in tool_ids: if get_loaded_tools_by_lineage: tools = toolbox.get_loaded_tools_by_lineage( tool_id ) @@ -83,7 +88,7 @@ message=message, status=status, redirect=redirect ) ) - params = util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers + params = galaxy.util.Params( kwd, sanitize = False ) #Sanitize parameters when substituting into command line via input wrappers #do param translation here, used by datasource tools if tool.input_translator: tool.input_translator.translate( params ) @@ -103,7 +108,7 @@ toolbox=self.get_toolbox(), tool_version_select_field=tool_version_select_field, tool=tool, - util=util, + util=galaxy.util, add_frame=add_frame, **vars ) @@ -215,7 +220,7 @@ #create an incoming object from the original job's dataset-modified param objects incoming = {} params_to_incoming( incoming, tool.inputs, params_objects, trans.app ) - incoming[ "tool_state" ] = util.object_to_string( state.encode( tool, trans.app ) ) + incoming[ "tool_state" ] = galaxy.util.object_to_string( state.encode( tool, trans.app ) ) template, vars = tool.handle_input( trans, incoming, old_errors=upgrade_messages ) #update new state with old parameters # Is the "add frame" stuff neccesary here? add_frame = AddFrameData() @@ -228,7 +233,7 @@ toolbox=self.get_toolbox(), tool_version_select_field=tool_version_select_field, tool=tool, - util=util, + util=galaxy.util, add_frame=add_frame, tool_id_version_message=tool_id_version_message, **vars ) @@ -288,9 +293,9 @@ tool = self.get_toolbox().get_tool( tool_id ) if not tool: return False # bad tool_id - nonfile_params = util.Params( kwd, sanitize=False ) + nonfile_params = galaxy.util.Params( kwd, sanitize=False ) if kwd.get( 'tool_state', None ) not in ( None, 'None' ): - encoded_state = util.string_to_object( kwd["tool_state"] ) + encoded_state = galaxy.util.string_to_object( kwd["tool_state"] ) tool_state = DefaultToolState() tool_state.decode( encoded_state, tool, trans.app ) else: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: sort: update log to browser
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3317f0d7f848/ changeset: 3317f0d7f848 user: guerler date: 2013-03-08 18:10:19 summary: sort: update log to browser affected #: 1 file diff -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 -r 3317f0d7f848445cff6b384bf5a1915db677d139 tools/filters/sorter.py --- a/tools/filters/sorter.py +++ b/tools/filters/sorter.py @@ -35,15 +35,15 @@ # retrieve options input = options.input output = options.output - key = ["-k" + k for k in options.key] + key = [" -k" + k for k in options.key] # grep comments grep_comments = "(grep '^#' %s) > %s" % (input, output) - print grep_comments + #print grep_comments # grep and sort columns sort_columns = "(grep '^[^#]' %s | sort -f -t '\t' %s) >> %s" % (input, ' '.join(key), output) - print sort_columns + #print sort_columns # execute os.system(grep_comments) Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: guerler: sort: preserves comments in sorted output
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7f28b9616f70/ changeset: 7f28b9616f70 user: guerler date: 2013-03-08 17:52:37 summary: sort: preserves comments in sorted output affected #: 5 files diff -r 7fe3612562eea76387e88ab7d184fccc3160cb43 -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 test-data/sort_in1.bed --- a/test-data/sort_in1.bed +++ b/test-data/sort_in1.bed @@ -1,3 +1,6 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, diff -r 7fe3612562eea76387e88ab7d184fccc3160cb43 -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 test-data/sort_out1.bed --- a/test-data/sort_out1.bed +++ b/test-data/sort_out1.bed @@ -1,29 +1,32 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters +chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, +chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, +chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, +chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, +chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, +chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, +chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, +chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, +chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, +chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, +chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, +chr6 108722790 108950942 NM_145315 0 + 108722976 108950321 0 13 325,224,52,102,131,100,59,83,71,101,141,114,750, 0,28931,52094,60760,61796,71339,107102,152319,181970,182297,215317,224802,227402, +chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, +chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, -chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, -chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, -chr6 108722790 108950942 NM_145315 0 + 108722976 108950321 0 13 325,224,52,102,131,100,59,83,71,101,141,114,750, 0,28931,52094,60760,61796,71339,107102,152319,181970,182297,215317,224802,227402, +chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, +chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, +chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, chr7 113320332 113924911 AK131266 0 + 113862563 113893433 0 20 285,91,178,90,58,75,138,51,201,178,214,105,88,84,77,102,122,70,164,1124, 0,201692,340175,448290,451999,484480,542213,543265,543478,545201,556083,558358,565876,567599,573029,573245,575738,577123,577946,603455, -chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, -chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, -chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, -chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, -chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, chr8 118880786 119193239 NM_000127 0 - 118881131 119192466 0 11 531,172,161,90,96,119,133,120,108,94,1735, 0,5355,7850,13505,19068,20309,23098,30863,36077,37741,310718, +chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, +chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, +chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, +chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, +chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, +chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, +chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, -chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, -chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, -chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, -chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, -chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, -chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, -chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, -chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, -chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, -chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, -chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, -chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, -chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, -chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, -chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, -chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, -chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, diff -r 7fe3612562eea76387e88ab7d184fccc3160cb43 -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 test-data/sort_out2.bed --- a/test-data/sort_out2.bed +++ b/test-data/sort_out2.bed @@ -1,3 +1,18 @@ +# comment 1 \n\n''" again +# comment 2 **}"''' special +# comment 3 @n/n""" characters +chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, +chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, +chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, +chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, +chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, +chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, +chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, +chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, +chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, +chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, +chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, chr6 108298214 108386086 NM_007214 0 - 108299600 108385906 0 21 1530,105,99,102,159,174,60,83,148,155,93,133,95,109,51,59,62,113,115,100,304, 0,2490,6246,10831,12670,23164,23520,27331,31052,32526,34311,36130,36365,38609,41028,42398,43048,51479,54500,59097,87568, chr6 108593954 108616704 NM_003269 0 + 108594662 108615360 0 9 733,146,88,236,147,97,150,106,1507, 0,5400,8778,10445,12037,14265,14749,15488,21243, chr6 108639410 108689143 NM_152827 0 - 108640045 108688818 0 3 741,125,487, 0,2984,49246, @@ -6,24 +21,12 @@ chr7 116511232 116557294 NM_003391 0 - 116512159 116556994 0 5 1157,265,278,227,383, 0,20384,37843,43339,45679, chr7 116713967 116902666 NM_000492 0 + 116714099 116901113 0 27 185,111,109,216,90,164,126,247,93,183,192,95,87,724,129,38,251,80,151,228,101,249,156,90,173,106,1754, 0,24290,29071,50936,54313,55285,56585,60137,62053,68678,79501,107776,110390,111971,114967,122863,123569,126711,130556,131618,134650,147559,162475,172879,184725,185496,186945, chr7 116944658 117107512 AF377960 0 - 116945541 116979926 0 23 1129,102,133,64,186,206,179,188,153,100,87,80,96,276,118,255,151,100,204,1654,225,108,173, 0,7364,8850,10413,13893,14398,17435,24259,24615,35177,35359,45901,47221,49781,56405,66857,69787,72208,73597,80474,100111,150555,162681, -chr2 118288484 118306183 NM_006773 0 + 118288583 118304530 0 14 184,285,144,136,101,200,115,140,162,153,114,57,178,1796, 0,2765,4970,6482,6971,7183,7468,9890,10261,10768,11590,14270,14610,15903, -chr2 118389378 118390700 BC005078 0 - 118390395 118390500 0 1 1322, 0, chr8 118880786 119193239 NM_000127 0 - 118881131 119192466 0 11 531,172,161,90,96,119,133,120,108,94,1735, 0,5355,7850,13505,19068,20309,23098,30863,36077,37741,310718, -chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, chr9 128763240 128783870 NM_174933 0 + 128764156 128783586 0 12 261,118,74,159,76,48,56,63,129,117,127,370, 0,522,875,5630,12374,12603,15040,15175,18961,19191,20037,20260, chr9 128787362 128789566 NM_014908 0 - 128787519 128789136 0 1 2204, 0, chr9 128789530 128848928 NM_015354 0 + 128789552 128848511 0 44 54,55,74,85,81,45,93,120,212,115,201,90,66,120,127,153,127,88,77,115,121,67,129,140,107,207,170,70,68,196,78,86,146,182,201,93,159,138,75,228,132,74,130,594, 0,1491,5075,8652,9254,10312,11104,11317,20808,21702,23060,25462,31564,32908,33566,34851,35204,35595,35776,37202,38860,39111,39891,40349,42422,45499,45827,46675,47158,47621,50453,50840,51474,51926,53831,54186,55119,55619,57449,57605,57947,58352,58541,58804, chr9 128849867 128870133 NM_020145 0 - 128850516 128869987 0 11 757,241,101,90,24,63,93,134,129,142,209, 0,1071,1736,2085,2635,4201,6376,6736,13056,14247,20057, -chr5 131170738 131357870 AF099740 0 - 131311206 131357817 0 31 112,124,120,81,65,40,120,129,61,88,94,79,72,102,144,117,89,73,96,135,135,78,74,52,33,179,100,102,65,115,248, 0,11593,44117,47607,104668,109739,114675,126366,135488,137518,138009,140437,152389,153373,155388,159269,160793,162981,164403,165577,166119,167611,169501,178260,179675,180901,181658,182260,182953,183706,186884, -chr5 131424245 131426795 NM_000588 0 + 131424298 131426383 0 5 215,42,90,42,535, 0,313,1658,1872,2015, -chr5 131556201 131590458 NM_004199 0 - 131556601 131582218 0 15 471,97,69,66,54,100,71,177,194,240,138,152,97,100,170, 0,2316,2802,5596,6269,11138,11472,15098,16528,17674,21306,24587,25142,25935,34087, -chr5 131621285 131637046 NM_003687 0 + 131621326 131635821 0 7 134,152,82,179,164,118,1430, 0,4915,8770,13221,13609,14097,14331, -chr1 147962006 147975713 NM_005997 0 - 147962192 147975670 0 6 574,145,177,115,153,160, 0,1543,7859,9048,9340,13547, -chr1 147984101 148035079 BC007833 0 + 147984545 148033414 0 14 529,32,81,131,118,153,300,206,84,49,85,130,46,1668, 0,25695,28767,33118,33695,33998,35644,38005,39629,40577,41402,43885,48367,49310, -chr1 148077485 148111797 NM_002651 0 - 148078400 148111728 0 12 1097,121,133,266,124,105,110,228,228,45,937,77, 0,2081,2472,6871,9907,10257,11604,14199,15637,18274,23636,34235, -chr1 148185113 148187485 NM_002796 0 + 148185136 148187378 0 7 163,207,147,82,117,89,120, 0,416,877,1199,1674,1977,2252, +chrX 122719582 122773357 NM_001167 0 + 122745047 122766566 0 7 96,909,100,79,43,201,6985, 0,25433,28421,31040,32533,40295,46790, chrX 152648233 152662158 NM_000425 0 - 152648964 152662138 0 28 963,12,73,135,156,120,174,123,202,116,223,71,198,111,125,157,167,112,144,132,185,112,171,123,203,106,11,100, 0,1436,1545,1951,2390,2653,2889,3156,3367,3772,4717,5122,5424,5868,6066,6370,6629,6909,7588,7871,8124,8456,8858,9125,10220,10660,11296,13825, chrX 152691216 152693487 NM_000054 0 + 152691446 152693029 0 3 255,885,664, 0,616,1607, chrX 152693677 152712545 NM_001666 0 - 152694029 152712503 0 22 586,100,93,184,74,234,106,135,78,61,103,28,85,192,102,222,129,183,63,163,205,109, 0,1693,2066,2364,2635,2794,3129,3323,3545,3752,5323,5647,5841,6032,6401,11455,11778,13249,13719,13987,14227,18759, -chr2 220108603 220116964 NM_001927 0 + 220108689 220116217 0 9 664,61,96,162,126,221,44,83,789, 0,1718,1874,2118,2451,2963,5400,7286,7572, -chr2 220229182 220233943 NM_024536 0 - 220229609 220233765 0 4 1687,180,574,492, 0,1990,2660,4269, diff -r 7fe3612562eea76387e88ab7d184fccc3160cb43 -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 tools/filters/sorter.py --- a/tools/filters/sorter.py +++ b/tools/filters/sorter.py @@ -1,49 +1,59 @@ """ -Sorts tabular data on one or more columns. + Sorts tabular data on one or more columns. All comments of the file are collected + and placed at the beginning of the sorted output file. + + usage: sorter.py [options] + -i, --input: Tabular file to be sorted + -o, --output: Sorted output file + -k, --key: Key (see manual for bash/sort) + + usage: sorter.py input output [key ...] +""" +# 03/05/2013 guerler -usage: %prog [options] - -i, --input=i: Tabular file to be sorted - -o, --out_file1=o: Sorted output file - -c, --column=c: First column to sort on - -s, --style=s: Sort style (numerical or alphabetical) - -r, --order=r: Order (ASC or DESC) +# imports +import os, re, string, sys +from optparse import OptionParser -usage: %prog input out_file1 column style order [column style ...] -""" - -import os, re, string, sys -from galaxy import eggs -import pkg_resources; pkg_resources.require( "bx-python" ) -from bx.cookbook import doc_optparse - +# error def stop_err( msg ): sys.stderr.write( "%s\n" % msg ) sys.exit() +# main def main(): - #Parse Command Line - options, args = doc_optparse.parse( __doc__ ) + # define options + parser = OptionParser() + parser.add_option("-i", "--input") + parser.add_option("-o", "--output") + parser.add_option("-k", "--key", action="append") + + # parse + options, args = parser.parse_args() + try: - inputfile = options.input - outputfile = '-o %s' % options.out_file1 - columns = [options.column] - styles = [('','n')[options.style == 'num']] - orders = [('','r')[options.order == 'DESC']] - col_style_orders = sys.argv[6:] - if len(col_style_orders) > 1: - columns.extend([col_style_orders[i] for i in range(0,len(col_style_orders),3)]) - styles.extend([('','n')[col_style_orders[i] == 'num'] for i in range(1,len(col_style_orders),3)]) - orders.extend([('','r')[col_style_orders[i] == 'DESC'] for i in range(2,len(col_style_orders),3)]) - cols = [ '-k%s,%s%s%s'%(columns[i], columns[i], styles[i], orders[i]) for i in range(len(columns)) ] + # retrieve options + input = options.input + output = options.output + key = ["-k" + k for k in options.key] + + # grep comments + grep_comments = "(grep '^#' %s) > %s" % (input, output) + print grep_comments + + # grep and sort columns + sort_columns = "(grep '^[^#]' %s | sort -f -t '\t' %s) >> %s" % (input, ' '.join(key), output) + print sort_columns + + # execute + os.system(grep_comments) + os.system(sort_columns) + except Exception, ex: - stop_err('Error parsing input parameters\n' + str(ex)) + stop_err('Error running sorter.py\n' + str(ex)) - # Launch sort. - cmd = "sort -f -t ' ' %s %s %s" % (' '.join(cols), outputfile, inputfile) - try: - os.system(cmd) - except Exception, ex: - stop_err('Error running sort command\n' + str(ex)) + # exit + sys.exit(0) if __name__ == "__main__": main() diff -r 7fe3612562eea76387e88ab7d184fccc3160cb43 -r 7f28b9616f70d4c6515c6fd88061374a2b01a283 tools/filters/sorter.xml --- a/tools/filters/sorter.xml +++ b/tools/filters/sorter.xml @@ -1,130 +1,61 @@ -<tool id="sort1" name="Sort" version="1.0.1"> - <description>data in ascending or descending order</description> - <command interpreter="python"> - sorter.py - --input=$input - --out_file1=$out_file1 - --column=$column - --style=$style - --order=$order - #for $col in $column_set: - ${col.other_column} - ${col.other_style} - ${col.other_order} - #end for - </command> - <inputs> - <param format="tabular" name="input" type="data" label="Sort Dataset" /> - <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/> - <param name="style" type="select" label="with flavor"> - <option value="num">Numerical sort</option> - <option value="alpha">Alphabetical sort</option> - </param> - <param name="order" type="select" label="everything in"> - <option value="DESC">Descending order</option> - <option value="ASC">Ascending order</option> - </param> - <repeat name="column_set" title="Column selection"> - <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /> - <param name="other_style" type="select" label="with flavor"> - <option value="num">Numerical sort</option> - <option value="alpha">Alphabetical sort</option> - </param> - <param name="other_order" type="select" label="everything in"> - <option value="DESC">Descending order</option> - <option value="ASC">Ascending order</option> - </param> - </repeat> - </inputs> - <outputs> - <data format="input" name="out_file1" metadata_source="input"/> - </outputs> - <tests> - <test> - <param name="input" value="sort_in1.bed"/> - <param name="column" value="1"/> - <param name="style" value="num"/> - <param name="order" value="ASC"/> - <param name="other_column" value="3"/> - <param name="other_style" value="num"/> - <param name="other_order" value="ASC"/> - <output name="out_file1" file="sort_out1.bed"/> - </test> - <test> - <param name="input" value="sort_in1.bed"/> - <param name="column" value="3"/> - <param name="style" value="alpha"/> - <param name="order" value="ASC"/> - <param name="other_column" value="1"/> - <param name="other_style" value="alpha"/> - <param name="other_order" value="ASC"/> - <output name="out_file1" file="sort_out2.bed"/> - </test> - </tests> - <help> +<tool id="sort1" name="Sort" version="1.0.2"> + <description>data in ascending or descending order</description> + <command interpreter="python"> + sorter.py + --input=$input + --output=$output + --key=$column,$column$style$order + #for $col in $column_set: + --key=${col.other_column},${col.other_column}${col.other_style}${col.other_order} + #end for + </command> + <inputs> + <param format="tabular" name="input" type="data" label="Sort Dataset" /> + <param name="column" label="on column" type="data_column" data_ref="input" accept_default="true"/> + <param name="style" type="select" label="with flavor"> + <option value="n">Numerical sort</option> + <option value="">Alphabetical sort</option> + </param> + <param name="order" type="select" label="everything in"> + <option value="r">Descending order</option> + <option value="">Ascending order</option> + </param> + <repeat name="column_set" title="Column selection"> + <param name="other_column" label="on column" type="data_column" data_ref="input" accept_default="true" /> + <param name="other_style" type="select" label="with flavor"> + <option value="n">Numerical sort</option> + <option value="">Alphabetical sort</option> + </param> + <param name="other_order" type="select" label="everything in"> + <option value="r">Descending order</option> + <option value="">Ascending order</option> + </param> + </repeat> + </inputs> + <outputs> + <data format="input" name="output" metadata_source="input"/> + </outputs> + <tests> + <test> + <param name="input" value="sort_in1.bed"/> + <param name="column" value="1"/> + <param name="style" value=""/> + <param name="order" value=""/> + <param name="other_column" value="3"/> + <param name="other_style" value="n"/> + <param name="other_order" value="r"/> + <output name="output" file="sort_out1.bed"/> + </test> + <test> + <param name="input" value="sort_in1.bed"/> + <param name="column" value="1"/> + <param name="style" value=""/> + <param name="order" value=""/> + <param name="other_column" value="3"/> + <param name="other_style" value="n"/> + <param name="other_order" value=""/> + <output name="output" file="sort_out2.bed"/> + </test> + </tests> -.. class:: infomark - -**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* - ------ - -**Syntax** - -This tool sorts the dataset on any number of columns in either ascending or descending order. - -* Numerical sort orders numbers by their magnitude, ignores all characters besides numbers, and evaluates a string of numbers to the value they signify. -* Alphabetical sort is a phonebook type sort based on the conventional order of letters in an alphabet. Each nth letter is compared with the nth letter of other words in the list, starting at the first letter of each word and advancing to the second, third, fourth, and so on, until the order is established. Therefore, in an alphabetical sort, 2 comes after 100 (1 < 2). - ------ - -**Examples** - -The list of numbers 4,17,3,5 collates to 3,4,5,17 by numerical sorting, while it collates to 17,3,4,5 by alphabetical sorting. - -Sorting the following:: - - Q d 7 II jhu 45 - A kk 4 I h 111 - Pd p 1 ktY WS 113 - A g 10 H ZZ 856 - A edf 4 tw b 234 - BBB rt 10 H ZZ 100 - A rew 10 d b 1111 - C sd 19 YH aa 10 - Hah c 23 ver bb 467 - MN gtr 1 a X 32 - N j 9 a T 205 - BBB rrf 10 b Z 134 - odfr ws 6 Weg dew 201 - C f 3 WW SW 34 - A jhg 4 I b 345 - Pd gf 7 Gthe de 567 - rS hty 90 YY LOp 89 - A g 10 H h 43 - A g 4 I h 500 - -on columns 1 (alpha), 3 (num), and 6 (num) in ascending order will yield:: - - A kk 4 I h 111 - A edf 4 tw b 234 - A jhg 4 I b 345 - A g 4 I h 500 - A g 10 H h 43 - A g 10 H ZZ 856 - A rew 10 d b 1111 - BBB rt 10 H ZZ 100 - BBB rrf 10 b Z 134 - C f 3 WW SW 34 - C sd 19 YH aa 10 - Hah c 23 ver bb 467 - MN gtr 1 a X 32 - N j 9 a T 205 - odfr ws 6 Weg dew 201 - Pd p 1 ktY WS 113 - Pd gf 7 Gthe de 567 - Q d 7 II jhu 45 - rS hty 90 YY LOp 89 - - </help></tool> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 3 new changesets
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d024cb15459d/ changeset: d024cb15459d user: Kyle Ellrott date: 2013-02-26 01:48:38 summary: Passing extra parameters from root index to tool index. This will allow for tools to pick up parameters from the request URL and map them with the request_param_translation, even when inside the root frame. affected #: 2 files diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r d024cb15459def8c539b3b894eeac94b6bd95ca3 lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -22,7 +22,8 @@ return trans.fill_template( "root/index.mako", tool_id=tool_id, workflow_id=workflow_id, - m_c=m_c, m_a=m_a ) + m_c=m_c, m_a=m_a, + params=kwd ) ## ---- Tool related ----------------------------------------------------- diff -r 26d7b89d9f897a28876214c88faed215973e6fcd -r d024cb15459def8c539b3b894eeac94b6bd95ca3 templates/webapps/galaxy/root/index.mako --- a/templates/webapps/galaxy/root/index.mako +++ b/templates/webapps/galaxy/root/index.mako @@ -118,7 +118,7 @@ if trans.app.config.require_login and not trans.user: center_url = h.url_for( controller='user', action='login' ) elif tool_id is not None: - center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True ) + center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True, **params ) elif workflow_id is not None: center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) elif m_c is not None: https://bitbucket.org/galaxy/galaxy-central/commits/9fca1c2e2b5d/ changeset: 9fca1c2e2b5d user: Kyle Ellrott date: 2013-02-26 20:09:44 summary: For external tool requests, without a 'URL' parameter but with other parameters, still process the incoming parameters. This will allow for tools that have 'request_param_translation' tables without the 'URL' parameter mapping to accept external requests and fill in their tool form page. affected #: 1 file diff -r d024cb15459def8c539b3b894eeac94b6bd95ca3 -r 9fca1c2e2b5db9853db093c241276c5431355b98 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1809,6 +1809,8 @@ if "runtool_btn" not in incoming and "URL" not in incoming: if not self.display_interface: return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) + if len(incoming): + self.update_state( trans, self.inputs_by_page[state.page], state.inputs, incoming, old_errors=old_errors or {} ) return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): https://bitbucket.org/galaxy/galaxy-central/commits/7fe3612562ee/ changeset: 7fe3612562ee user: dannon date: 2013-03-08 16:35:38 summary: Merged in kellrott/galaxy-central (pull request #130) Passing extra parameters from root index to tool index. affected #: 3 files diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r 7fe3612562eea76387e88ab7d184fccc3160cb43 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1922,6 +1922,8 @@ if "runtool_btn" not in incoming and "URL" not in incoming: if not self.display_interface: return 'message.mako', dict( status='info', message="The interface for this tool cannot be displayed", refresh_frames=['everything'] ) + if len(incoming): + self.update_state( trans, self.inputs_by_page[state.page], state.inputs, incoming, old_errors=old_errors or {} ) return "tool_form.mako", dict( errors={}, tool_state=state, param_values={}, incoming={} ) # Process incoming data if not( self.check_values ): diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r 7fe3612562eea76387e88ab7d184fccc3160cb43 lib/galaxy/webapps/galaxy/controllers/root.py --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -22,7 +22,8 @@ return trans.fill_template( "root/index.mako", tool_id=tool_id, workflow_id=workflow_id, - m_c=m_c, m_a=m_a ) + m_c=m_c, m_a=m_a, + params=kwd ) ## ---- Tool related ----------------------------------------------------- diff -r c7bde6953174eabe5510c89d023ffd8232592f11 -r 7fe3612562eea76387e88ab7d184fccc3160cb43 templates/webapps/galaxy/root/index.mako --- a/templates/webapps/galaxy/root/index.mako +++ b/templates/webapps/galaxy/root/index.mako @@ -118,7 +118,7 @@ if trans.app.config.require_login and not trans.user: center_url = h.url_for( controller='user', action='login' ) elif tool_id is not None: - center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True ) + center_url = h.url_for( 'tool_runner', tool_id=tool_id, from_noframe=True, **params ) elif workflow_id is not None: center_url = h.url_for( controller='workflow', action='run', id=workflow_id ) elif m_c is not None: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the hours_between_check config setting to handle float values in addition to int values. This allows for functional tests to test the Galaxy update manager.
by commits-noreply＠bitbucket.org 08 Mar '13

08 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/c7bde6953174/ changeset: c7bde6953174 user: greg date: 2013-03-08 15:14:23 summary: Enhance the hours_between_check config setting to handle float values in addition to int values. This allows for functional tests to test the Galaxy update manager. affected #: 2 files diff -r e3ae0bbd800a68532fc51625642e0abc29e1b085 -r c7bde6953174eabe5510c89d023ffd8232592f11 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -69,9 +69,14 @@ self.shed_tool_data_table_config = resolve_path( kwargs.get( 'shed_tool_data_table_config', 'shed_tool_data_table_conf.xml' ), self.root ) self.enable_tool_shed_check = string_as_bool( kwargs.get( 'enable_tool_shed_check', False ) ) try: - self.hours_between_check = int( kwargs.get( 'hours_between_check', 12 ) ) - if self.hours_between_check < 1 or self.hours_between_check > 24: - self.hours_between_check = 12 + self.hours_between_check = kwargs.get( 'hours_between_check', 12 ) + if isinstance( hours_between_check, float ): + # Float values are supported for functional tests. + if self.hours_between_check < 0.001 or self.hours_between_check > 24.0: + self.hours_between_check = 12.0 + else: + if self.hours_between_check < 1 or self.hours_between_check > 24: + self.hours_between_check = 12 except: self.hours_between_check = 12 self.update_integrated_tool_panel = kwargs.get( "update_integrated_tool_panel", True ) diff -r e3ae0bbd800a68532fc51625642e0abc29e1b085 -r c7bde6953174eabe5510c89d023ffd8232592f11 lib/tool_shed/galaxy_install/update_manager.py --- a/lib/tool_shed/galaxy_install/update_manager.py +++ b/lib/tool_shed/galaxy_install/update_manager.py @@ -18,7 +18,7 @@ self.sleeper = Sleeper() self.restarter = threading.Thread( target=self.__restarter ) self.restarter.start() - self.seconds_to_sleep = app.config.hours_between_check * 3600 + self.seconds_to_sleep = int( app.config.hours_between_check * 3600 ) def __restarter( self ): log.info( 'Update manager restarter starting up...' ) while self.running: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: browser tests: fix broken debug statement
by commits-noreply＠bitbucket.org 07 Mar '13

07 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/e3ae0bbd800a/ changeset: e3ae0bbd800a user: carlfeberhard date: 2013-03-07 01:45:31 summary: browser tests: fix broken debug statement affected #: 1 file diff -r 68455758f27e572cd637fc2b014773f0dfa9002d -r e3ae0bbd800a68532fc51625642e0abc29e1b085 test/casperjs/modules/historypanel.js --- a/test/casperjs/modules/historypanel.js +++ b/test/casperjs/modules/historypanel.js @@ -161,7 +161,7 @@ //precondition: historyItemWrapper's (hda dom elements) should have an id // we could use the selector directly, but better if it errors before an attempted delete var hdaId = spaceghost.getElementInfo( hdaSelector ).attributes.id; - spaceghost.debug( 'hda id: ' + spaceghost.jsonStr( hdaId ) ); + spaceghost.debug( 'hda id: ' + hdaId ); // get the delete icon and click it //TODO: handle disabled delete icon? Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 07 Mar '13

07 Mar '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/a601d530ec95/ changeset: a601d530ec95 user: carlfeberhard date: 2013-03-07 01:35:29 summary: api: add exception type to history_contents, index errors affected #: 1 file diff -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 -r a601d530ec9577eace628940dd4a385aebb803f3 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -73,8 +73,8 @@ # don't fail entire list if hda err's, record and move on # (making sure http recvr knows it's err'd) trans.response.status = 500 - log.error( "Error in history API at listing contents " + - "with history %s, hda %s: %s", history_id, encoded_hda_id, str( exc ) ) + log.error( "Error in history API at listing contents with history %s, hda %s: (%s) %s", + history_id, encoded_hda_id, type( exc ), str( exc ) ) rval.append( self._exception_as_hda_dict( trans, encoded_hda_id, exc ) ) else: https://bitbucket.org/galaxy/galaxy-central/commits/68455758f27e/ changeset: 68455758f27e user: carlfeberhard date: 2013-03-07 01:36:25 summary: objectstore: add messages to ObjectNot* exceptions raised affected #: 1 file diff -r a601d530ec9577eace628940dd4a385aebb803f3 -r 68455758f27e572cd637fc2b014773f0dfa9002d lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -731,7 +731,8 @@ if self.exists(obj, **kwargs): return bool(self.size(obj, **kwargs) > 0) else: - raise ObjectNotFound() + raise ObjectNotFound( 'objectstore.empty, object does not exist: %s, kwargs: %s' + %( str( obj ), str( kwargs ) ) ) def size(self, obj, **kwargs): rel_path = self._construct_path(obj, **kwargs) @@ -816,7 +817,8 @@ # even if it does not exist. # if dir_only: # return cache_path - raise ObjectNotFound() + raise ObjectNotFound( 'objectstore.get_filename, no cache_path: %s, kwargs: %s' + %( str( obj ), str( kwargs ) ) ) # return cache_path # Until the upload tool does not explicitly create the dataset, return expected path def update_from_file(self, obj, file_name=None, create=False, **kwargs): @@ -841,7 +843,8 @@ # Update the file on S3 self._push_to_os(rel_path, source_file) else: - raise ObjectNotFound() + raise ObjectNotFound( 'objectstore.update_from_file, object does not exist: %s, kwargs: %s' + %( str( obj ), str( kwargs ) ) ) def get_object_url(self, obj, **kwargs): if self.exists(obj, **kwargs): @@ -947,7 +950,8 @@ try: obj.object_store_id = random.choice(self.weighted_backend_ids) except IndexError: - raise ObjectInvalid() + raise ObjectInvalid( 'objectstore.create, could not generate obj.object_store_id: %s, kwargs: %s' + %( str( obj ), str( kwargs ) ) ) object_session( obj ).add( obj ) object_session( obj ).flush() log.debug("Selected backend '%s' for creation of %s %s" % (obj.object_store_id, obj.__class__.__name__, obj.id)) @@ -984,7 +988,8 @@ if object_store_id is not None: return self.backends[object_store_id].__getattribute__(method)(obj, **kwargs) if default_is_exception: - raise default() + raise default( 'objectstore, __call_method failed: %s on %s, kwargs: %s' + %( method, str( obj ), str( kwargs ) ) ) else: return default @@ -1073,4 +1078,3 @@ calling_format=calling_format, path=config.os_conn_path) return s3_conn - Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: inithello: Added more information to the tool test environment dict. Restrict when do_not_test flag is set. Made test logging format more structured. Moved some utility code to the util module.
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7677b8c3ef43/ changeset: 7677b8c3ef43 user: inithello date: 2013-03-06 22:07:32 summary: Added more information to the tool test environment dict. Restrict when do_not_test flag is set. Made test logging format more structured. Moved some utility code to the util module. affected #: 3 files diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 lib/tool_shed/scripts/check_repositories_for_functional_tests.py --- a/lib/tool_shed/scripts/check_repositories_for_functional_tests.py +++ b/lib/tool_shed/scripts/check_repositories_for_functional_tests.py @@ -12,9 +12,9 @@ from galaxy import eggs import pkg_resources -pkg_resources.require( "SQLAlchemy >= 0.4" ) -pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +eggs.require( "SQLAlchemy >= 0.4" ) +eggs.require( 'mercurial' ) +from mercurial import hg, ui, commands, __version__ import time, ConfigParser, shutil from datetime import datetime, timedelta @@ -24,12 +24,13 @@ import galaxy.webapps.tool_shed.config as tool_shed_config import galaxy.webapps.tool_shed.model.mapping import sqlalchemy as sa -from galaxy.model.orm import and_, not_, distinct +from galaxy.model.orm import and_, not_ from galaxy.util.json import from_json_string, to_json_string from galaxy.web import url_for +from galaxy.tools import parameters from tool_shed.util.shed_util_common import clone_repository, get_configured_ui -from base.util import get_test_environment +from base.util import get_test_environment, get_database_version, get_repository_current_revision assert sys.version_info[:2] >= ( 2, 4 ) @@ -50,6 +51,7 @@ now = strftime( "%Y-%m-%d %H:%M:%S" ) print "#############################################################################" print "# %s - Checking repositories for tools with functional tests." % now + print "# This tool shed is configured to listen on %s:%s." % ( config_parser.get( 'server:main', 'host' ), config_parser.get( 'server:main', 'port' ) ) app = FlagRepositoriesApplication( config ) if options.info_only: @@ -113,7 +115,7 @@ } ''' start = time.time() - repository_ids_to_check = [] + checked_repository_ids = [] tool_count = 0 has_tests = 0 no_tests = 0 @@ -121,25 +123,31 @@ # Get the list of metadata records to check for functional tests and test data. Limit this to records that have not been flagged do_not_test # or tools_functionally_correct. Also filter out changeset revisions that are not downloadable, because it's redundant to test a revision that # a user can't install. + # Initialize the repository_status dict with the test environment, but leave the test_errors empty. + repository_status = {} + repository_status[ 'test_environment' ] = get_test_environment() + repository_status[ 'test_environment' ][ 'tool_shed_database_version' ] = get_database_version( app ) + repository_status[ 'test_environment' ][ 'tool_shed_mercurial_version' ] = __version__.version + repository_status[ 'test_environment' ][ 'tool_shed_revision' ] = get_repository_current_revision( os.getcwd() ) + repository_status[ 'test_errors' ] = [] metadata_records_to_check = app.sa_session.query( app.model.RepositoryMetadata ) \ .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, app.model.RepositoryMetadata.table.c.do_not_test == False, app.model.RepositoryMetadata.table.c.tools_functionally_correct == False ) ) \ .all() for metadata_record in metadata_records_to_check: + repository_status[ 'test_errors' ] = [] name = metadata_record.repository.name owner = metadata_record.repository.user.username changeset_revision = str( metadata_record.changeset_revision ) - repository_status = {} + if metadata_record.repository.id not in checked_repository_ids: + checked_repository_ids.append( metadata_record.repository.id ) # If this changeset revision has no tools, we don't need to do anything here, the install and test script has a filter for returning # only repositories that contain tools. if 'tools' not in metadata_record.metadata: no_tools += 1 continue else: - # Initialize the repository_status dict with the test environment, but leave the test_errors empty. - repository_status[ 'test_environment' ] = get_test_environment() - repository_status[ 'test_errors' ] = [] # Loop through all the tools in this metadata record, checking each one for defined functional tests. for tool_metadata in metadata_record.metadata[ 'tools' ]: tool_count += 1 @@ -153,12 +161,9 @@ if 'tests' not in tool_metadata or not tool_metadata[ 'tests' ]: if verbose: print '# No functional tests defined for %s.' % tool_id - if 'test_errors' not in repository_status: - repository_status[ 'test_errors' ] = [] - test_id = 'Functional tests for %s' % tool_id # The repository_metadata.tool_test_errors attribute should always have the following structure: # { - # "environment": + # "test_environment": # { # "python_version": "2.7.2", # "architecture": "x86_64", @@ -168,15 +173,16 @@ # [ # { # "test_id": "Something that will easily identify what the problem is", - # "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." # }, # ] # } - # Optionally, "stderr" and "traceback" may be included in a test_errors dict, if they are relevant. - test_errors = dict( test_id=test_id, - stdout='No functional tests defined in changeset revision %s of repository %s owned by %s.' % \ - ( changeset_revision, name, owner ) ) + # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. + test_id = 'Functional tests for %s' % tool_id + test_errors = dict( stderr='No functional tests defined for tool %s in changeset revision %s of repository %s owned by %s.' % \ + ( tool_id, changeset_revision, name, owner ) ) repository_status[ 'test_errors' ].append( test_errors ) + repository_status[ 'status' ] = 'failed' no_tests += 1 else: has_tests += 1 @@ -201,14 +207,12 @@ if 'test-data' in dirs: has_test_data = True break - # Remove the cloned path. + # Remove the cloned repository path. if os.path.exists( work_dir ): shutil.rmtree( work_dir ) if not has_test_data: if verbose: print '# Test data missing in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) - repository_status[ 'test_environment' ] = get_test_environment() - test_id = 'Find functional test data for %s' % metadata_record.repository.name # The repository_metadata.tool_test_errors attribute should always have the following structure: # { # "test_environment": @@ -221,28 +225,40 @@ # [ # { # "test_id": "Something that will easily identify what the problem is", - # "stdout": "The output of the test, or a more detailed description of what was tested and why it failed." + # "stderr": "The output of the test, or a more detailed description of what was tested and why it failed." # }, # ] # } - # Optionally, "stderr" and "traceback" may be included in a test_errors dict, if they are relevant. - test_errors = dict( test_id=test_id, - stdout='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) + # Optionally, "stdout" and "traceback" may be included in a test_errors dict, if they are relevant. + test_id = 'Find functional test data for %s' % metadata_record.repository.name + test_errors = dict( stderr='No test data found for changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) ) repository_status[ 'test_errors' ].append( test_errors ) + repository_status[ 'status' ] = 'failed' else: if verbose: print '# Test data found in changeset revision %s of repository %s owned by %s.' % ( changeset_revision, name, owner ) if not info_only: + # If repository_status[ 'test_errors' ] is empty, no issues were found, and we can just update time_last_tested with the platform + # on which this script was run. if repository_status[ 'test_errors' ]: - metadata_record.do_not_test = True + # If functional test definitions or test data are missing, set do_not_test = True if and only if: + # a) There are multiple downloadable revisions, and the revision being tested is not the most recent downloadable revision. In this case, + # the revision will never be updated with correct data, and re-testing it would be redundant. + # b) There are one or more downloadable revisions, and the revision being tested is the most recent downloadable revision. In this case, if + # the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will be replaced + # with the new changeset revision, which will be automatically tested. + if should_set_do_not_test_flag( app, metadata_record.repository, changeset_revision ): + metadata_record.do_not_test = True metadata_record.tools_functionally_correct = False - metadata_record.tool_test_errors = to_json_string( repository_status ) + else: + repository_status[ 'status' ] = 'passed' + metadata_record.tool_test_errors = repository_status metadata_record.time_last_tested = datetime.utcnow() app.sa_session.add( metadata_record ) app.sa_session.flush() stop = time.time() - print '# Checked %d tools in %d changeset revisions.' % ( tool_count, len( metadata_records_to_check ) ) - print '# Found %d changeset revisions without tools.' % no_tools + print '# Checked %d repositories with %d tools in %d changeset revisions.' % ( len( checked_repository_ids ), tool_count, len( metadata_records_to_check ) ) + print '# Skipped %d changeset revisions without tools.' % no_tools print '# Found %d tools without functional tests.' % no_tests print '# Found %d tools with functional tests.' % has_tests if info_only: @@ -250,6 +266,52 @@ print "# Elapsed time: ", stop - start print "#############################################################################" +def get_repo_changelog_tuples( repo_path ): + repo = hg.repository( ui.ui(), repo_path ) + changelog_tuples = [] + for changeset in repo.changelog: + ctx = repo.changectx( changeset ) + changelog_tuples.append( ( ctx.rev(), str( ctx ) ) ) + return changelog_tuples + +def is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ): + # Get a list of ( numeric revision, changeset hash ) tuples from the changelog. + changelog = get_repo_changelog_tuples( repository.repo_path( app ) ) + latest_downloadable_revision = None + for ctx_rev, changeset_hash in changelog: + if changeset_hash in downloadable_revisions: + # The last changeset hash in the changelog that is present in the list of downloadable revisions will always be the most + # recent downloadable revision, since the changelog tuples are ordered from earliest to most recent. + latest_downloadable_revision = changeset_hash + if latest_downloadable_revision == changeset_revision: + return True + return False + +def should_set_do_not_test_flag( app, repository, changeset_revision ): + ''' + Returns True if: + a) There are multiple downloadable revisions, and the provided changeset revision is not the most recent downloadable revision. In this case, + the revision will never be updated with correct data, and re-testing it would be redundant. + b) There are one or more downloadable revisions, and the provided changeset revision is the most recent downloadable revision. In this case, if + the repository is updated with test data or functional tests, the downloadable changeset revision that was tested will be replaced + with the new changeset revision, which will be automatically tested. + ''' + metadata_records = app.sa_session.query( app.model.RepositoryMetadata ) \ + .filter( and_( app.model.RepositoryMetadata.table.c.downloadable == True, + app.model.RepositoryMetadata.table.c.repository_id == repository.id ) ) \ + .all() + downloadable_revisions = [ metadata_record.changeset_revision for metadata_record in metadata_records ] + is_latest_revision = is_most_recent_downloadable_revision( app, repository, changeset_revision, downloadable_revisions ) + if len( downloadable_revisions ) == 1: + return True + elif len( downloadable_revisions ) > 1 and is_latest_revision: + return True + elif len( downloadable_revisions ) > 1 and not is_latest_revision: + return True + else: + return False + + class FlagRepositoriesApplication( object ): """Encapsulates the state of a Universe application""" def __init__( self, config ): diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 test/base/util.py --- a/test/base/util.py +++ b/test/base/util.py @@ -12,6 +12,44 @@ sys.path = new_path from galaxy.util import parse_xml +from galaxy import eggs + +eggs.require( 'mercurial' ) + +from mercurial import hg, ui, commands + +def get_repository_current_revision( repo_path ): + ''' + This method uses the python mercurial API to get the current working directory's mercurial changeset hash. Note that if the author of mercurial + changes the API, this method will have to be updated or replaced. + ''' + # Initialize a mercurial repo object from the provided path. + repo = hg.repository( ui.ui(), repo_path ) + # Get the working directory's change context. + ctx = repo[ None ] + # Extract the changeset hash of the first parent of that change context (the most recent changeset to which the working directory was updated). + changectx = ctx.parents()[ 0 ] + # Also get the numeric revision, so we can return the customary id:hash changeset identifiers. + ctx_rev = changectx.rev() + hg_id = '%d:%s' % ( ctx_rev, str( changectx ) ) + return hg_id + +def get_database_version( app ): + ''' + This method returns the value of the version column from the migrate_version table, using the provided app's SQLAlchemy session to determine + which table to get that from. This way, it's provided with an instance of a Galaxy UniverseApplication, it will return the Galaxy instance's + database migration version. If a tool shed UniverseApplication is provided, it returns the tool shed's database migration version. + ''' + sa_session = app.model.context.current + result = sa_session.execute( 'SELECT version FROM migrate_version LIMIT 1' ) + # This query will return the following structure: + # row = [ column 0, column 1, ..., column n ] + # rows = [ row 0, row 1, ..., row n ] + # The first column in the first row is the version number we want. + for row in result: + version = row[ 0 ] + break + return version def get_installed_repository_info( elem, last_galaxy_test_file_dir, last_tested_repository_name, last_tested_changeset_revision, tool_path ): """ diff -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 -r 7677b8c3ef43d06eda783f1c08ec521d2bd625e5 test/install_and_test_tool_shed_repositories/functional_tests.py --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -61,7 +61,7 @@ import nose.loader import nose.plugins.manager -from base.util import parse_tool_panel_config +from base.util import parse_tool_panel_config, get_database_version, get_test_environment, get_repository_current_revision from common import update @@ -105,7 +105,7 @@ # Define a default location to find the list of repositories to check. galaxy_repository_list = os.environ.get( 'GALAXY_INSTALL_TEST_REPOSITORY_LIST_LOCATIOM', 'repository_list.json' ) -galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', 'http://toolshed.local:10001' ) +galaxy_tool_shed_url = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_URL', 'http://localhost:9009' ) tool_shed_api_key = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_SHED_API_KEY', None ) assert tool_shed_api_key is not None, 'Unable to proceed without API key.' @@ -191,25 +191,22 @@ else: raise AssertonError( 'Unknown format %s.' % format ) -def get_test_environment(): - rval = {} - rval[ 'python_version' ] = platform.python_version() - rval[ 'architecture' ] = platform.machine() - os, hostname, os_version, uname, arch, processor = platform.uname() - rval[ 'system' ] = '%s %s' % ( os, os_version ) - return rval - def json_from_url( url ): url_handle = urllib.urlopen( url ) url_contents = url_handle.read() return from_json_string( url_contents ) -def register_test_failure( url, metadata_id, test_errors ): - params = dict( tools_functionally_correct='false', do_not_test='true', tool_test_errors=test_errors ) - return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) - -def register_test_success( url, metadata_id ): - params = dict( tools_functionally_correct='true', do_not_test='true' ) +def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ): + params = {} + if tests_passed: + params[ 'tools_functionally_correct' ] = 'true' + params[ 'do_not_test' ] = 'true' + test_results_dict[ 'status' ] = 'passed' + else: + params[ 'tools_functionally_correct' ] = 'false' + params[ 'do_not_test' ] = 'true' + test_results_dict[ 'status' ] = 'failed' + params[ 'tool_test_errors' ] = test_results_dict return update( tool_shed_api_key, '%s' % ( url_join( galaxy_tool_shed_url, 'api', 'repository_revisions', metadata_id ) ), params, return_formatted=False ) def run_tests( test_config ): @@ -233,8 +230,8 @@ # Initialize some variables for the summary that will be printed to stdout. repositories_tested = 0 - repositories_passed = 0 - repositories_failed = 0 + repositories_passed = [] + repositories_failed = [] tool_path = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_PATH', 'tools' ) if 'HTTP_ACCEPT_LANGUAGE' not in os.environ: @@ -395,6 +392,9 @@ log.info( "Repositories will be installed from the tool shed at %s" % galaxy_tool_shed_url ) success = False repository_status = dict() + test_environment = get_test_environment() + test_environment[ 'galaxy_database_version' ] = get_database_version( app ) + test_environment[ 'galaxy_revision'] = get_repository_current_revision( os.getcwd() ) try: # Iterate through a list of repository info dicts. log.info( "Retrieving repositories to install from the URL:\n%s\n" % str( galaxy_tool_shed_url ) ) @@ -421,6 +421,7 @@ "url": "/api/repository_revisions/529fd61ab1c6cc36" } """ + repository_status = dict() repository_id = repository_to_install_dict.get( 'repository_id', None ) changeset_revision = repository_to_install_dict.get( 'changeset_revision', None ) metadata_revision_id = repository_to_install_dict.get( 'id', None ) @@ -460,6 +461,8 @@ "user_id": "529fd61ab1c6cc36" } """ + name = repository_dict[ 'name' ] + owner = repository_dict[ 'owner' ] # Use the repository information dict to generate an install method that will install the repository into the embedded # Galaxy application, with tool dependencies and repository dependencies, if any. test_install_repositories.generate_install_method( repository_dict ) @@ -489,13 +492,13 @@ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ], repository_dict[ 'owner' ] ) ) repository_status[ 'test_errors' ] = [ test_errors ] # Record the status of this repository in the tool shed. - register_test_failure( galaxy_tool_shed_url, metadata_revision_id, repository_status ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. execute_uninstall_method( repository_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox - repositories_failed += 1 + repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) else: # If the repository does have a test-data directory, we write the generated shed_tools_dict to a file, so the functional # test framework can find it. @@ -519,16 +522,17 @@ success = result.wasSuccessful() # Record some information about the environment in which this test was run, in case a failure is specific to a certain processor # architecture or operating system. - repository_dict[ 'test_environment' ] = get_test_environment() - repository_dict[ 'functional_tests_passed' ] = success + repository_dict[ 'test_environment' ] = test_environment test_errors = [] if success: # This repository's tools passed all functional tests. Update the repository_metadata table in the tool shed's database - # to reflect that. Call the register_test_success method, which executes a PUT request to the repository_revisions API + # to reflect that. Call the register_test_result method, which executes a PUT request to the repository_revisions API # controller with the status of the test. This also sets the do_not_test and tools_functionally correct flags, and # updates the time_last_tested field to today's date. - repositories_passed += 1 - register_test_success( galaxy_tool_shed_url, metadata_revision_id ) + repositories_passed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + repository_status[ 'test_environment' ] = test_environment + repository_status[ 'test_errors' ] = [] + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=True ) log.debug( 'Revision %s of repository %s installed and passed functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) else: @@ -569,13 +573,13 @@ test_errors.append( test_status ) if test_errors: # Only update test_errors for this repository if it's not empty. - repository_status[ 'test_environment' ] = get_test_environment() + repository_status[ 'test_environment' ] = test_environment repository_status[ 'test_errors' ] = test_errors - # Call the register_test_failure method, which executes a PUT request to the repository_revisions API controller with the failure - # status of the test, and updates tool_test_errors with the relevant log data. + # Call the register_test_result method, which executes a PUT request to the repository_revisions API controller with the outcome + # status of the tests, and updates tool_test_errors with the relevant log data. # This also sets the do_not_test and tools_functionally correct flags, and updates the time_last_tested field to today's date. - repositories_failed += 1 - register_test_failure( galaxy_tool_shed_url, metadata_revision_id, repository_status ) + repositories_failed.append( dict( name=name, owner=owner, changeset_revision=changeset_revision ) ) + register_test_result( galaxy_tool_shed_url, metadata_revision_id, repository_status, tests_passed=False ) log.debug( 'Revision %s of repository %s installed successfully, but did not pass functional tests.' % \ ( repository_dict[ 'changeset_revision' ], repository_dict[ 'name' ] ) ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the @@ -618,8 +622,14 @@ print "# %s - repository installation and testing script completed." % now print "# Repository revisions tested: %d" % repositories_tested if repositories_tested > 0: - print "# Repository revisions passed: %d" % repositories_passed - print "# Repository revisions failed: %d" % repositories_failed + if repositories_passed: + print "# Repositories passed:" + for repository in repositories_passed: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) + if repositories_failed: + print "# Repositories failed:" + for repository in repositories_failed: + print "# %s owned by %s, changeset revision %s" % ( repository[ 'name' ], repository[ 'owner' ], repository[ 'changeset_revision' ] ) print "####################################################################################" if success: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: fix imports
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/48dad8860ee1/ changeset: 48dad8860ee1 user: carlfeberhard date: 2013-03-06 21:34:25 summary: fix imports affected #: 1 file diff -r dc10f9430b6aaaf85c67a998d4ce666a5b66b600 -r 48dad8860ee1b1196a1f242c9dd18d218e6db573 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -19,6 +19,9 @@ eggs.require( "simplejson" ) eggs.require( "MarkupSafe" ) #MarkupSafe must load before mako eggs.require( "Mako" ) +eggs.require( "elementtree" ) +eggs.require( "Paste" ) +eggs.require( "SQLAlchemy >= 0.4" ) import simplejson from cgi import FieldStorage Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/dc10f9430b6a/ changeset: dc10f9430b6a user: dan date: 2013-03-06 19:47:00 summary: Fix display bug for DrillDown Fields and parameters. affected #: 2 files diff -r d1b435e7200d4543fe3060ba9c362220d73d615b -r dc10f9430b6aaaf85c67a998d4ce666a5b66b600 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1075,9 +1075,9 @@ >>> print p.get_html() <div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span><input type="checkbox" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="some_name" value="option1" >Option 1 </div> @@ -1085,9 +1085,9 @@ <input type="checkbox" name="some_name" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span><input type="checkbox" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="some_name" value="option3" >Option 3 </div> @@ -1121,9 +1121,9 @@ >>> print p.get_html() <div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span><input type="radio" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="some_name" value="option1" >Option 1 </div> @@ -1131,9 +1131,9 @@ <input type="radio" name="some_name" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span><input type="radio" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="some_name" value="option3" >Option 3 </div> diff -r d1b435e7200d4543fe3060ba9c362220d73d615b -r dc10f9430b6aaaf85c67a998d4ce666a5b66b600 lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -396,9 +396,9 @@ >>> print t.get_html() <div class="form-row drilldown-container" id="drilldown--666f6f"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span><input type="checkbox" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="foo" value="option1" >Option 1 </div> @@ -406,9 +406,9 @@ <input type="checkbox" name="foo" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span><input type="checkbox" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="foo" value="option3" >Option 3 </div> @@ -427,9 +427,9 @@ >>> print t.get_html() <div class="form-row drilldown-container" id="drilldown--666f6f"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span><input type="radio" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="foo" value="option1" >Option 1 </div> @@ -437,9 +437,9 @@ <input type="radio" name="foo" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span><input type="radio" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="foo" value="option3" >Option 3 </div> @@ -501,10 +501,10 @@ html.append( '<div class="form-row-input">') drilldown_group_id = "%s-%s" % ( base_id, hexlify( option['value'] ) ) if option['options']: - html.append( '<span class="%s" id="%s-click"></span>' % ( span_class, drilldown_group_id ) ) + html.append( '<div><span class="%s" id="%s-click"></span>' % ( span_class, drilldown_group_id ) ) html.append( '<input type="%s" name="%s%s" value="%s" %s>%s' % ( self.display, prefix, self.name, escaped_option_value, selected, option['name']) ) if option['options']: - html.append( '<div class="form-row" id="%s-container" style="float: left; margin-left: 1em;">' % ( drilldown_group_id ) ) + html.append( '</div><div class="form-row" id="%s-container" style="float: left; margin-left: 1em;">' % ( drilldown_group_id ) ) recurse_options( html, option['options'], drilldown_group_id, expanded_options ) html.append( '</div>') html.append( '</div>') https://bitbucket.org/galaxy/galaxy-central/commits/779b54139013/ changeset: 779b54139013 branch: stable user: dan date: 2013-03-06 19:47:00 summary: Fix display bug for DrillDown Fields and parameters. affected #: 2 files diff -r b65ed2831d234ef9fc78b8c15d9dfcf40f879c4f -r 779b54139013946613772d02a3fbb61e067f3267 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1072,9 +1072,9 @@ >>> print p.get_html() <div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span><input type="checkbox" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="some_name" value="option1" >Option 1 </div> @@ -1082,9 +1082,9 @@ <input type="checkbox" name="some_name" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span><input type="checkbox" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="some_name" value="option3" >Option 3 </div> @@ -1118,9 +1118,9 @@ >>> print p.get_html() <div class="form-row drilldown-container" id="drilldown--736f6d655f6e616d65"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-click"></span><input type="radio" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="some_name" value="option1" >Option 1 </div> @@ -1128,9 +1128,9 @@ <input type="radio" name="some_name" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-click"></span><input type="radio" name="some_name" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--736f6d655f6e616d65-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="some_name" value="option3" >Option 3 </div> diff -r b65ed2831d234ef9fc78b8c15d9dfcf40f879c4f -r 779b54139013946613772d02a3fbb61e067f3267 lib/galaxy/web/form_builder.py --- a/lib/galaxy/web/form_builder.py +++ b/lib/galaxy/web/form_builder.py @@ -396,9 +396,9 @@ >>> print t.get_html() <div class="form-row drilldown-container" id="drilldown--666f6f"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span><input type="checkbox" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="foo" value="option1" >Option 1 </div> @@ -406,9 +406,9 @@ <input type="checkbox" name="foo" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span><input type="checkbox" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="checkbox" name="foo" value="option3" >Option 3 </div> @@ -427,9 +427,9 @@ >>> print t.get_html() <div class="form-row drilldown-container" id="drilldown--666f6f"><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-click"></span><input type="radio" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="foo" value="option1" >Option 1 </div> @@ -437,9 +437,9 @@ <input type="radio" name="foo" value="option2" >Option 2 </div><div class="form-row-input"> - <span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span> + <div><span class="form-toggle icon-button toggle-expand" id="drilldown--666f6f-68656164696e6731-68656164696e6731-click"></span><input type="radio" name="foo" value="heading1" >Heading 1 - <div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"> + </div><div class="form-row" id="drilldown--666f6f-68656164696e6731-68656164696e6731-container" style="float: left; margin-left: 1em;"><div class="form-row-input"><input type="radio" name="foo" value="option3" >Option 3 </div> @@ -501,10 +501,10 @@ html.append( '<div class="form-row-input">') drilldown_group_id = "%s-%s" % ( base_id, hexlify( option['value'] ) ) if option['options']: - html.append( '<span class="%s" id="%s-click"></span>' % ( span_class, drilldown_group_id ) ) + html.append( '<div><span class="%s" id="%s-click"></span>' % ( span_class, drilldown_group_id ) ) html.append( '<input type="%s" name="%s%s" value="%s" %s>%s' % ( self.display, prefix, self.name, escaped_option_value, selected, option['name']) ) if option['options']: - html.append( '<div class="form-row" id="%s-container" style="float: left; margin-left: 1em;">' % ( drilldown_group_id ) ) + html.append( '</div><div class="form-row" id="%s-container" style="float: left; margin-left: 1em;">' % ( drilldown_group_id ) ) recurse_options( html, option['options'], drilldown_group_id, expanded_options ) html.append( '</div>') html.append( '</div>') Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Re-add secondary workflow api download route to support applications already using it.
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d1b435e7200d/ changeset: d1b435e7200d user: dannon date: 2013-03-06 19:09:35 summary: Re-add secondary workflow api download route to support applications already using it. affected #: 1 file diff -r 47480f11d5d5c99dfe67710cc4a7ecbaeebebb4a -r d1b435e7200d4543fe3060ba9c362220d73d615b lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -153,6 +153,8 @@ # Defines a named route "import_workflow". webapp.api_mapper.connect("import_workflow", "/api/workflows/upload", controller="workflows", action="import_new_workflow", conditions=dict(method=["POST"])) webapp.api_mapper.connect("workflow_dict", '/api/workflows/{workflow_id}/download', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) + # Preserve the following download route for now for dependent applications -- deprecate at some point + webapp.api_mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) # Connect logger from app if app.trace_logger: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/47480f11d5d5/ changeset: 47480f11d5d5 user: natefoo date: 2013-03-06 18:40:31 summary: Make maf-to-bed object-store aware affected #: 1 file diff -r 3ec030c42b9e59234d4c005e6a7f1de6e0e1e736 -r 47480f11d5d5c99dfe67710cc4a7ecbaeebebb4a tools/maf/maf_to_bed_code.py --- a/tools/maf/maf_to_bed_code.py +++ b/tools/maf/maf_to_bed_code.py @@ -1,3 +1,4 @@ +import os import pkg_resources; pkg_resources.require( "bx-python" ) from bx.align import maf from galaxy import datatypes, config, jobs @@ -38,7 +39,8 @@ app.model.context.add( history ) app.model.context.flush() try: - move(filepath,newdata.file_name) + app.object_store.update_from_file( newdata.dataset, file_name=filepath, create=True ) + os.unlink( filepath ) newdata.info = newdata.name newdata.state = newdata.states.OK except: https://bitbucket.org/galaxy/galaxy-central/commits/b65ed2831d23/ changeset: b65ed2831d23 branch: stable user: natefoo date: 2013-03-06 18:40:31 summary: Make maf-to-bed object-store aware affected #: 1 file diff -r c177960e4ed61925a8b6c858e1f3f8d54c93cb37 -r b65ed2831d234ef9fc78b8c15d9dfcf40f879c4f tools/maf/maf_to_bed_code.py --- a/tools/maf/maf_to_bed_code.py +++ b/tools/maf/maf_to_bed_code.py @@ -1,3 +1,4 @@ +import os import pkg_resources; pkg_resources.require( "bx-python" ) from bx.align import maf from galaxy import datatypes, config, jobs @@ -38,7 +39,8 @@ app.model.context.add( history ) app.model.context.flush() try: - move(filepath,newdata.file_name) + app.object_store.update_from_file( newdata.dataset, file_name=filepath, create=True ) + os.unlink( filepath ) newdata.info = newdata.name newdata.state = newdata.states.OK except: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Remove unused import in objectstore.
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3ec030c42b9e/ changeset: 3ec030c42b9e user: dannon date: 2013-03-06 18:38:48 summary: Remove unused import in objectstore. affected #: 1 file diff -r 3bdfcb0a7e9722ce4074c52528a4c0324f2fb83a -r 3ec030c42b9e59234d4c005e6a7f1de6e0e1e736 lib/galaxy/objectstore/__init__.py --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -9,7 +9,6 @@ import time import random import shutil -import statvfs import logging import threading import subprocess Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Fix incorrect workflow download path via the API. Now is is correctly set to /api/workflows/<id>/download
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3bdfcb0a7e97/ changeset: 3bdfcb0a7e97 user: dannon date: 2013-03-06 17:50:53 summary: Fix incorrect workflow download path via the API. Now is is correctly set to /api/workflows/<id>/download affected #: 2 files diff -r f97ebc1cf57d20019b785106221964931b122bb7 -r 3bdfcb0a7e9722ce4074c52528a4c0324f2fb83a lib/galaxy/webapps/galaxy/api/workflows.py --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -258,24 +258,16 @@ trans.sa_session.flush() return rval - # ---------------------------------------------------------------------------------------------- # - # ---------------------------------------------------------------------------------------------- # - # ---- RPARK EDITS ---- # - # ---------------------------------------------------------------------------------------------- # - # ---------------------------------------------------------------------------------------------- # @web.expose_api - #(a)web.json def workflow_dict( self, trans, workflow_id, **kwd ): """ GET /api/workflows/{encoded_workflow_id}/download Returns a selected workflow as a json dictionary. """ - try: stored_workflow = trans.sa_session.query(self.app.model.StoredWorkflow).get(trans.security.decode_id(workflow_id)) except Exception,e: return ("Workflow with ID='%s' can not be found\n Exception: %s") % (workflow_id, str( e )) - # check to see if user has permissions to selected workflow if stored_workflow.user != trans.user and not trans.user_is_admin(): if trans.sa_session.query(trans.app.model.StoredWorkflowUserShareAssociation).filter_by(user=trans.user, stored_workflow=stored_workflow).count() == 0: diff -r f97ebc1cf57d20019b785106221964931b122bb7 -r 3bdfcb0a7e9722ce4074c52528a4c0324f2fb83a lib/galaxy/webapps/galaxy/buildapp.py --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -152,7 +152,7 @@ # "POST /api/workflows/import" => ``workflows.import_workflow()``. # Defines a named route "import_workflow". webapp.api_mapper.connect("import_workflow", "/api/workflows/upload", controller="workflows", action="import_new_workflow", conditions=dict(method=["POST"])) - webapp.api_mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) + webapp.api_mapper.connect("workflow_dict", '/api/workflows/{workflow_id}/download', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) # Connect logger from app if app.trace_logger: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: dannon: Workflow controller import refactoring to eliminate import *'s.
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/f97ebc1cf57d/ changeset: f97ebc1cf57d user: dannon date: 2013-03-06 17:40:39 summary: Workflow controller import refactoring to eliminate import *'s. affected #: 1 file diff -r 7eda7f4b555eb13e8382e34b9739a38af6072849 -r f97ebc1cf57d20019b785106221964931b122bb7 lib/galaxy/webapps/galaxy/controllers/workflow.py --- a/lib/galaxy/webapps/galaxy/controllers/workflow.py +++ b/lib/galaxy/webapps/galaxy/controllers/workflow.py @@ -1,28 +1,41 @@ -from galaxy.web.base.controller import * - import pkg_resources pkg_resources.require( "simplejson" ) pkg_resources.require( "SVGFig" ) + +import base64 +import httplib +import math +import os +import sgmllib import simplejson -import base64, httplib, urllib2, sgmllib, svgfig, urllib, urllib2 -import math -from galaxy.web.framework.helpers import time_ago, grids -from galaxy.tools.parameters import * -from galaxy.tools import DefaultToolState -from galaxy.tools.parameters.grouping import Repeat, Conditional +import svgfig +import urllib2 + +from sqlalchemy import and_ +from tool_shed.util import encoding_util + +from galaxy import model +from galaxy import util +from galaxy import web from galaxy.datatypes.data import Data +from galaxy.jobs.actions.post import ActionBox +from galaxy.model.mapping import desc +from galaxy.tools.parameters import RuntimeValue, visit_input_values +from galaxy.tools.parameters.basic import DataToolParameter, DrillDownSelectToolParameter, SelectToolParameter, UnvalidatedValue +from galaxy.tools.parameters.grouping import Conditional, Repeat +from galaxy.util.json import from_json_string from galaxy.util.odict import odict from galaxy.util.sanitize_html import sanitize_html -from galaxy.util.topsort import topsort, topsort_levels, CycleError -from tool_shed.util import encoding_util -from galaxy.workflow.modules import * -from galaxy import model -from galaxy import util -from galaxy.model.mapping import desc -from galaxy.model.orm import * -from galaxy.model.item_attrs import * +from galaxy.util.topsort import CycleError, topsort, topsort_levels +from galaxy.web import error, url_for +from galaxy.web.base.controller import BaseUIController, SharableMixin, UsesStoredWorkflowMixin +from galaxy.model.item_attrs import UsesAnnotations, UsesItemRatings +from galaxy.web.framework import form +from galaxy.web.framework.helpers import grids, time_ago from galaxy.web.framework.helpers import to_unicode -from galaxy.jobs.actions.post import ActionBox +from galaxy.workflow.modules import module_factory, ToolModule + + class StoredWorkflowListGrid( grids.Grid ): class StepsColumn( grids.GridColumn ): @@ -1813,7 +1826,7 @@ # Unpack and add post-job actions. post_job_actions = step_dict.get( 'post_job_actions', {} ) for name, pja_dict in post_job_actions.items(): - pja = PostJobAction( pja_dict[ 'action_type' ], + pja = model.PostJobAction( pja_dict[ 'action_type' ], step, pja_dict[ 'output_name' ], pja_dict[ 'action_arguments' ] ) # Second pass to deal with connections between steps Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: carlfeberhard: history panel and api: if hda.peek is 'no peek' do not send peek through api and do not render peek in expanded hda
by commits-noreply＠bitbucket.org 06 Mar '13

06 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/7eda7f4b555e/ changeset: 7eda7f4b555e user: carlfeberhard date: 2013-03-06 16:02:24 summary: history panel and api: if hda.peek is 'no peek' do not send peek through api and do not render peek in expanded hda affected #: 3 files diff -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 -r 7eda7f4b555eb13e8382e34b9739a38af6072849 lib/galaxy/webapps/galaxy/api/history_contents.py --- a/lib/galaxy/webapps/galaxy/api/history_contents.py +++ b/lib/galaxy/webapps/galaxy/api/history_contents.py @@ -221,7 +221,8 @@ hda_dict[ 'display_types' ] = get_old_display_applications( trans, hda ) hda_dict[ 'visualizations' ] = hda.get_visualizations() - hda_dict[ 'peek' ] = to_unicode( hda.display_peek() ) + if hda.peek and hda.peek != 'no peek': + hda_dict[ 'peek' ] = to_unicode( hda.display_peek() ) if hda.creating_job and hda.creating_job.tool_id: tool_used = trans.app.toolbox.get_tool( hda.creating_job.tool_id ) diff -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 -r 7eda7f4b555eb13e8382e34b9739a38af6072849 static/scripts/mvc/dataset/hda-base.js --- a/static/scripts/mvc/dataset/hda-base.js +++ b/static/scripts/mvc/dataset/hda-base.js @@ -319,12 +319,13 @@ */ //TODO: curr. pre-formatted into table on the server side - may not be ideal/flexible _render_peek : function(){ - if( !this.model.get( 'peek' ) ){ return null; } + var peek = this.model.get( 'peek' ); + if( !peek ){ return null; } return $( '<div/>' ).append( $( '<pre/>' ) .attr( 'id', 'peek' + this.model.get( 'id' ) ) .addClass( 'peek' ) - .append( this.model.get( 'peek' ) ) + .append( peek ) ); }, diff -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 -r 7eda7f4b555eb13e8382e34b9739a38af6072849 static/scripts/packed/mvc/dataset/hda-base.js --- a/static/scripts/packed/mvc/dataset/hda-base.js +++ b/static/scripts/packed/mvc/dataset/hda-base.js @@ -1,1 +1,1 @@ -var HDABaseView=BaseView.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){if(a.logger){this.logger=this.model.logger=a.logger}this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urlTemplates=a.urlTemplates;this.expanded=a.expanded||false;this.model.bind("change",this.render,this)},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("<div/>").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);this.urls=this._renderUrls(this.urlTemplates,this.model.toJSON());a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this._setUpBehaviors(a);this.body=$(this._render_body());a.append(this.body);this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{try{c[f]=_.template(e,a)}catch(g){throw (b+"._renderUrls error: "+g+"\n rendering:"+e+"\n with "+JSON.stringify(a))}}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},_setUpBehaviors:function(a){a=a||this.$el;make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"})},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('<div class="historyItemTitleBar" style="overflow: hidden"></div>');a.append(this._render_titleButtons());a.append('<span class="state-icon"></span>');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("Display data in browser");a.href=this.urls.display}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("<div/>").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a)},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayApps:function(){if(!this.model.hasData()){return null}var a=$("<div/>").addClass("display-apps");if(!_.isEmpty(this.model.get("display_types"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_types")}))}if(!_.isEmpty(this.model.get("display_apps"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_apps")}))}return a},_render_peek:function(){if(!this.model.get("peek")){return null}return $("<div/>").append($("<pre/>").attr("id","peek"+this.model.get("id")).addClass("peek").append(this.model.get("peek")))},_render_body:function(){var a=$("<div/>").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: none");if(this.expanded){this._render_body_html(a);a.show()}return a},_render_body_html:function(a){a.html("");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NEW:break;case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.PAUSED:this._render_body_paused(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'))}a.append('<div style="clear: both"></div>');this._setUpBehaviors(a)},_render_body_not_viewable:function(a){a.append($("<div>"+_l("You do not have permission to view dataset")+".</div>"))},_render_body_uploading:function(a){a.append($("<div>"+_l("Dataset is uploading")+"</div>"))},_render_body_queued:function(a){a.append($("<div>"+_l("Job is waiting to run")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_paused:function(a){a.append($("<div>"+_l("Job is paused. Use the history menu to resume")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("<div>"+_l("Job is currently running")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("<div>"+this.model.get("misc_blurb")+"</div>"))}a.append((_l("An error occurred with this dataset")+": <i>"+$.trim(this.model.get("misc_info"))+"</i>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("<div>"+_l("The job creating this dataset was cancelled before completion")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("<div>"+_l("Metadata is being auto-detected")+".</div>"))},_render_body_empty:function(a){a.append($("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(_.extend(this.model.toJSON(),{urls:this.urls}))));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('<div class="clear"/>');a.append(this._render_displayApps());a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this;this.expanded=(a===undefined)?(!this.body.is(":visible")):(a);if(this.expanded){b._render_body_html(b.body);this.body.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{this.body.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},remove:function(b){var a=this;this.$el.fadeOut("fast",function(){a.$el.remove();if(b){b()}})},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetadata"],displayApps:Handlebars.templates["template-hda-displayApps"]}; \ No newline at end of file +var HDABaseView=BaseView.extend(LoggableMixin).extend({tagName:"div",className:"historyItemContainer",initialize:function(a){if(a.logger){this.logger=this.model.logger=a.logger}this.log(this+".initialize:",a);this.defaultPrimaryActionButtonRenderers=[this._render_showParamsButton];if(!a.urlTemplates){throw ("HDAView needs urlTemplates on initialize")}this.urlTemplates=a.urlTemplates;this.expanded=a.expanded||false;this.model.bind("change",this.render,this)},render:function(){var b=this,e=this.model.get("id"),c=this.model.get("state"),a=$("<div/>").attr("id","historyItem-"+e),d=(this.$el.children().size()===0);this.$el.attr("id","historyItemContainer-"+e);this.urls=this._renderUrls(this.urlTemplates,this.model.toJSON());a.addClass("historyItemWrapper").addClass("historyItem").addClass("historyItem-"+c);a.append(this._render_warnings());a.append(this._render_titleBar());this._setUpBehaviors(a);this.body=$(this._render_body());a.append(this.body);this.$el.fadeOut("fast",function(){b.$el.children().remove();b.$el.append(a).fadeIn("fast",function(){b.log(b+" rendered:",b.$el);var f="rendered";if(d){f+=":initial"}else{if(b.model.inReadyState()){f+=":ready"}}b.trigger(f)})});return this},_renderUrls:function(d,a){var b=this,c={};_.each(d,function(e,f){if(_.isObject(e)){c[f]=b._renderUrls(e,a)}else{if(f==="meta_download"){c[f]=b._renderMetaDownloadUrls(e,a)}else{try{c[f]=_.template(e,a)}catch(g){throw (b+"._renderUrls error: "+g+"\n rendering:"+e+"\n with "+JSON.stringify(a))}}}});return c},_renderMetaDownloadUrls:function(b,a){return _.map(a.meta_files,function(c){return{url:_.template(b,{id:a.id,file_type:c.file_type}),file_type:c.file_type}})},_setUpBehaviors:function(a){a=a||this.$el;make_popup_menus(a);a.find(".tooltip").tooltip({placement:"bottom"})},_render_warnings:function(){return $(jQuery.trim(HDABaseView.templates.messages(this.model.toJSON())))},_render_titleBar:function(){var a=$('<div class="historyItemTitleBar" style="overflow: hidden"></div>');a.append(this._render_titleButtons());a.append('<span class="state-icon"></span>');a.append(this._render_titleLink());return a},_render_titleButtons:function(){var a=$('<div class="historyItemButtons"></div>');a.append(this._render_displayButton());return a},_render_displayButton:function(){if((this.model.get("state")===HistoryDatasetAssociation.STATES.NOT_VIEWABLE)||(!this.model.get("accessible"))){this.displayButton=null;return null}var a={icon_class:"display",target:"galaxy_main"};if(this.model.get("purged")){a.enabled=false;a.title=_l("Cannot display datasets removed from disk")}else{a.title=_l("Display data in browser");a.href=this.urls.display}this.displayButton=new IconButtonView({model:new IconButton(a)});return this.displayButton.render().$el},_render_titleLink:function(){return $(jQuery.trim(HDABaseView.templates.titleLink(_.extend(this.model.toJSON(),{urls:this.urls}))))},_render_hdaSummary:function(){var a=_.extend(this.model.toJSON(),{urls:this.urls});return HDABaseView.templates.hdaSummary(a)},_render_primaryActionButtons:function(c){var a=this,b=$("<div/>").attr("id","primary-actions-"+this.model.get("id"));_.each(c,function(d){b.append(d.call(a))});return b},_render_downloadButton:function(){if(this.model.get("purged")||!this.model.hasData()){return null}var a=HDABaseView.templates.downloadLinks(_.extend(this.model.toJSON(),{urls:this.urls}));return $(a)},_render_showParamsButton:function(){this.showParamsButton=new IconButtonView({model:new IconButton({title:_l("View details"),href:this.urls.show_params,target:"galaxy_main",icon_class:"information"})});return this.showParamsButton.render().$el},_render_displayApps:function(){if(!this.model.hasData()){return null}var a=$("<div/>").addClass("display-apps");if(!_.isEmpty(this.model.get("display_types"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_types")}))}if(!_.isEmpty(this.model.get("display_apps"))){a.append(HDABaseView.templates.displayApps({displayApps:this.model.get("display_apps")}))}return a},_render_peek:function(){var a=this.model.get("peek");if(!a){return null}return $("<div/>").append($("<pre/>").attr("id","peek"+this.model.get("id")).addClass("peek").append(a))},_render_body:function(){var a=$("<div/>").attr("id","info-"+this.model.get("id")).addClass("historyItemBody").attr("style","display: none");if(this.expanded){this._render_body_html(a);a.show()}return a},_render_body_html:function(a){a.html("");switch(this.model.get("state")){case HistoryDatasetAssociation.STATES.NEW:break;case HistoryDatasetAssociation.STATES.NOT_VIEWABLE:this._render_body_not_viewable(a);break;case HistoryDatasetAssociation.STATES.UPLOAD:this._render_body_uploading(a);break;case HistoryDatasetAssociation.STATES.PAUSED:this._render_body_paused(a);break;case HistoryDatasetAssociation.STATES.QUEUED:this._render_body_queued(a);break;case HistoryDatasetAssociation.STATES.RUNNING:this._render_body_running(a);break;case HistoryDatasetAssociation.STATES.ERROR:this._render_body_error(a);break;case HistoryDatasetAssociation.STATES.DISCARDED:this._render_body_discarded(a);break;case HistoryDatasetAssociation.STATES.SETTING_METADATA:this._render_body_setting_metadata(a);break;case HistoryDatasetAssociation.STATES.EMPTY:this._render_body_empty(a);break;case HistoryDatasetAssociation.STATES.FAILED_METADATA:this._render_body_failed_metadata(a);break;case HistoryDatasetAssociation.STATES.OK:this._render_body_ok(a);break;default:a.append($('<div>Error: unknown dataset state "'+this.model.get("state")+'".</div>'))}a.append('<div style="clear: both"></div>');this._setUpBehaviors(a)},_render_body_not_viewable:function(a){a.append($("<div>"+_l("You do not have permission to view dataset")+".</div>"))},_render_body_uploading:function(a){a.append($("<div>"+_l("Dataset is uploading")+"</div>"))},_render_body_queued:function(a){a.append($("<div>"+_l("Job is waiting to run")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_paused:function(a){a.append($("<div>"+_l("Job is paused. Use the history menu to resume")+".</div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_running:function(a){a.append("<div>"+_l("Job is currently running")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_error:function(a){if(!this.model.get("purged")){a.append($("<div>"+this.model.get("misc_blurb")+"</div>"))}a.append((_l("An error occurred with this dataset")+": <i>"+$.trim(this.model.get("misc_info"))+"</i>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers.concat([this._render_downloadButton])))},_render_body_discarded:function(a){a.append("<div>"+_l("The job creating this dataset was cancelled before completion")+".</div>");a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_setting_metadata:function(a){a.append($("<div>"+_l("Metadata is being auto-detected")+".</div>"))},_render_body_empty:function(a){a.append($("<div>"+_l("No data")+": <i>"+this.model.get("misc_blurb")+"</i></div>"));a.append(this._render_primaryActionButtons(this.defaultPrimaryActionButtonRenderers))},_render_body_failed_metadata:function(a){a.append($(HDABaseView.templates.failedMetadata(_.extend(this.model.toJSON(),{urls:this.urls}))));this._render_body_ok(a)},_render_body_ok:function(a){a.append(this._render_hdaSummary());if(this.model.isDeletedOrPurged()){a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));return}a.append(this._render_primaryActionButtons([this._render_downloadButton,this._render_showParamsButton]));a.append('<div class="clear"/>');a.append(this._render_displayApps());a.append(this._render_peek())},events:{"click .historyItemTitle":"toggleBodyVisibility"},toggleBodyVisibility:function(c,a){var b=this;this.expanded=(a===undefined)?(!this.body.is(":visible")):(a);if(this.expanded){b._render_body_html(b.body);this.body.slideDown("fast",function(){b.trigger("body-expanded",b.model.get("id"))})}else{this.body.slideUp("fast",function(){b.trigger("body-collapsed",b.model.get("id"))})}},remove:function(b){var a=this;this.$el.fadeOut("fast",function(){a.$el.remove();if(b){b()}})},toString:function(){var a=(this.model)?(this.model+""):("(no model)");return"HDABaseView("+a+")"}});HDABaseView.templates={warningMsg:Handlebars.templates["template-warningmessagesmall"],messages:Handlebars.templates["template-hda-warning-messages"],titleLink:Handlebars.templates["template-hda-titleLink"],hdaSummary:Handlebars.templates["template-hda-hdaSummary"],downloadLinks:Handlebars.templates["template-hda-downloadLinks"],failedMetadata:Handlebars.templates["template-hda-failedMetadata"],displayApps:Handlebars.templates["template-hda-displayApps"]}; \ No newline at end of file Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: greg: Enhance the display of functional test results in the tol shed for tests run against contained tools when the repository is installed into Galaxy.
by commits-noreply＠bitbucket.org 05 Mar '13

05 Mar '13

1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/3a3eab0b0adb/ changeset: 3a3eab0b0adb user: greg date: 2013-03-05 22:37:51 summary: Enhance the display of functional test results in the tol shed for tests run against contained tools when the repository is installed into Galaxy. affected #: 4 files diff -r 26006e793a32a35328d941f6cea573b0108faab7 -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 lib/galaxy/webapps/tool_shed/controllers/repository.py --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -2452,7 +2452,6 @@ revision_label = suc.get_revision_label( trans, repository, changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, id, changeset_revision ) if repository_metadata: - repository_metadata_id = trans.security.encode_id( repository_metadata.id ) metadata = repository_metadata.metadata # Get a dictionary of all repositories upon which the contents of the current repository_metadata record depend. repository_dependencies = suc.get_repository_dependencies_for_changeset_revision( trans=trans, @@ -2468,7 +2467,6 @@ message += orphan_message status = 'warning' else: - repository_metadata_id = None metadata = None is_malicious = suc.changeset_is_malicious( trans, id, repository.tip( trans.app ) ) if is_malicious: @@ -2492,7 +2490,7 @@ return trans.fill_template( '/webapps/tool_shed/repository/view_repository.mako', repo=repo, repository=repository, - repository_metadata_id=repository_metadata_id, + repository_metadata=repository_metadata, metadata=metadata, containers_dict=containers_dict, avg_rating=avg_rating, diff -r 26006e793a32a35328d941f6cea573b0108faab7 -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako --- a/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako +++ b/templates/webapps/tool_shed/repository/display_tool_functional_test_results.mako @@ -5,6 +5,7 @@ <% from galaxy.web.framework.helpers import time_ago + from tool_shed.util.shed_util_common import to_safe_string changeset_revision = repository_metadata.changeset_revision has_metadata = repository.metadata_revisions @@ -157,23 +158,61 @@ <div class="form-row"><table class="grid"> %for test_results_dict in test_errors: + <% + test_id = test_results_dict.get( 'test_id', 'unknown' ) + if test_id != 'unknown': + # The test_id looks something like: + # test_tool_000003 (functional.test_toolbox.TestForTool_localhost:9009/repos/test/bwa_mappers/bwa_color_wrapper/1.0.2) + # Highlight the tool id and version. + test_id_items = test_id.split( '/' ) + tool_id = test_id_items[ -2 ] + tool_id = '<b>%s</b>' % tool_id + test_id_items[ -2 ] = tool_id + tool_version = test_id_items[ -1 ] + tool_version = '<b>%s</b>' % ( tool_version.rstrip( ')' ) ) + tool_version = '%s)' % tool_version + test_id_items[ -1 ] = tool_version + test_id = '/'.join( test_id_items ) + test_id_items = test_id.split( ' ' ) + test_num = test_id_items[ 0 ] + test_num = '<b>%s</b>' % test_num + test_id_items[ 0 ] = test_num + test_id = ' '.join( test_id_items ) + + test_status = '<font color="green">Test passed</font>' + stdout = test_results_dict.get( 'stdout', '' ) + if stdout: + stdout = to_safe_string( stdout, to_html=True ) + stderr = test_results_dict.get( 'stderr', '' ) + if stderr: + stderr = to_safe_string( stderr, to_html=True ) + test_status = '<font color="red">Test failed</font>' + traceback = test_results_dict.get( 'traceback', '' ) + if traceback: + traceback = to_safe_string( traceback, to_html=True ) + test_status = '<font color="red">Test failed</font>' + %><tr><td bgcolor="#FFFFCC"><b>Test id</b></td> - <td bgcolor="#FFFFCC">${test_results_dict.get( 'test_id', 'unknown' )}</td> + <td bgcolor="#FFFFCC">${test_id}</td> + </tr> + <tr> + <td><b>Status</b></td> + <td>${test_status}</td></tr> %if repository_metadata.tools_functionally_correct: <tr><td><b>Stdout</b></td> - <td>${test_results_dict.get( 'stdout', '' )}</td> + <td>${stdout}</td></tr> %else: <tr><td><b>Stderr</b></td> - <td>${test_results_dict.get( 'stderr', '' )}</td> + <td>${stderr}</td></tr><tr><td><b>Traceback</b></td> - <td>${test_results_dict.get( 'traceback', '' )}</td> + <td>${traceback}</td></tr> %endif %endfor diff -r 26006e793a32a35328d941f6cea573b0108faab7 -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 templates/webapps/tool_shed/repository/manage_repository.mako --- a/templates/webapps/tool_shed/repository/manage_repository.mako +++ b/templates/webapps/tool_shed/repository/manage_repository.mako @@ -88,6 +88,9 @@ %if can_browse_repository_reviews: <a class="action-button" href="${h.url_for( controller='repository_review', action='manage_repository_reviews', id=trans.app.security.encode_id( repository.id ) )}">Browse reviews of this repository</a> %endif + %if can_display_tool_functional_test_results: + <a class="action-button" href="${h.url_for( controller='repository', action='display_tool_functional_test_results', repository_id=trans.security.encode_id( repository.id ), repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )}">View tool functional test results</a> + %endif %if can_upload: <a class="action-button" href="${h.url_for( controller='upload', action='upload', repository_id=trans.security.encode_id( repository.id ) )}">Upload files to repository</a> %endif @@ -112,9 +115,6 @@ %if can_undeprecate: <a class="action-button" href="${h.url_for( controller='repository', action='deprecate', id=trans.security.encode_id( repository.id ), mark_deprecated=False )}">Mark repository as not deprecated</a> %endif - %if can_display_tool_functional_test_results: - <a class="action-button" href="${h.url_for( controller='repository', action='display_tool_functional_test_results', repository_id=trans.security.encode_id( repository.id ), repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )}">View tool functional test results</a> - %endif %if can_download: <a class="action-button" href="${h.url_for( controller='repository', action='download', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=changeset_revision, file_type='gz' )}">Download as a .tar.gz file</a><a class="action-button" href="${h.url_for( controller='repository', action='download', repository_id=trans.app.security.encode_id( repository.id ), changeset_revision=changeset_revision, file_type='bz2' )}">Download as a .tar.bz2 file</a> diff -r 26006e793a32a35328d941f6cea573b0108faab7 -r 3a3eab0b0adbda30e72fdda0c5de899728bcd5e6 templates/webapps/tool_shed/repository/view_repository.mako --- a/templates/webapps/tool_shed/repository/view_repository.mako +++ b/templates/webapps/tool_shed/repository/view_repository.mako @@ -25,6 +25,14 @@ can_view_change_log = trans.webapp.name == 'tool_shed' and not is_new changeset_revision_is_repository_tip = changeset_revision == repository.tip( trans.app ) + if repository_metadata: + if repository_metadata.includes_tools and repository_metadata.tool_test_errors is not None: + can_display_tool_functional_test_results = True + else: + can_display_tool_functional_test_results = False + else: + can_display_tool_functional_test_results = False + if can_push: browse_label = 'Browse or delete repository tip files' else: @@ -80,6 +88,9 @@ %if can_browse_repository_reviews: <a class="action-button" href="${h.url_for( controller='repository_review', action='manage_repository_reviews', id=trans.app.security.encode_id( repository.id ) )}">Browse reviews of this repository</a> %endif + %if can_display_tool_functional_test_results: + <a class="action-button" href="${h.url_for( controller='repository', action='display_tool_functional_test_results', repository_id=trans.security.encode_id( repository.id ), repository_metadata_id=trans.security.encode_id( repository_metadata.id ) )}">View tool functional test results</a> + %endif %if can_upload: <a class="action-button" href="${h.url_for( controller='upload', action='upload', repository_id=trans.security.encode_id( repository.id ) )}">Upload files to repository</a> %endif Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0

commit/galaxy-central: 2 new changesets
by commits-noreply＠bitbucket.org 05 Mar '13

05 Mar '13

2 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/1da2068fa58c/ changeset: 1da2068fa58c user: dannon date: 2013-03-05 22:15:54 summary: Fix workflow modules imports. affected #: 1 file diff -r f7904535003fdf8e56588fb78ebcdf9af913cc55 -r 1da2068fa58c5f1ac9d4bee510644a1dce20e814 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1,13 +1,20 @@ +""" +Modules used in building workflows +""" + +import logging import re + from elementtree.ElementTree import Element + +import galaxy.tools from galaxy import web -from galaxy.tools.parameters import DataToolParameter, DummyDataset, RuntimeValue, check_param, visit_input_values -import galaxy.tools +from galaxy.jobs.actions.post import ActionBox +from galaxy.model import PostJobAction +from galaxy.tools.parameters import check_param, DataToolParameter, DummyDataset, RuntimeValue, visit_input_values from galaxy.util.bunch import Bunch from galaxy.util.json import from_json_string, to_json_string -from galaxy.jobs.actions.post import ActionBox -from galaxy.model import PostJobAction -import logging + log = logging.getLogger( __name__ ) https://bitbucket.org/galaxy/galaxy-central/commits/26006e793a32/ changeset: 26006e793a32 user: dannon date: 2013-03-05 22:18:16 summary: Workflow modules: Adjust spacing to new guidelines. affected #: 1 file diff -r 1da2068fa58c5f1ac9d4bee510644a1dce20e814 -r 26006e793a32a35328d941f6cea573b0108faab7 lib/galaxy/workflow/modules.py --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -15,8 +15,8 @@ from galaxy.util.bunch import Bunch from galaxy.util.json import from_json_string, to_json_string +log = logging.getLogger( __name__ ) -log = logging.getLogger( __name__ ) class WorkflowModule( object ): @@ -31,6 +31,7 @@ Create a new instance of the module with default state """ return Class( trans ) + @classmethod def from_dict( Class, trans, d ): """ @@ -38,6 +39,7 @@ dictionary `d`. """ return Class( trans ) + @classmethod def from_workflow_step( Class, trans, step ): return Class( trans ) @@ -51,10 +53,13 @@ def get_type( self ): return self.type + def get_name( self ): return self.name + def get_tool_id( self ): return None + def get_tooltip( self, static_path='' ): return None @@ -62,14 +67,19 @@ def get_state( self ): return None + def get_errors( self ): return None + def get_data_inputs( self ): return [] + def get_data_outputs( self ): return [] + def update_state( self ): pass + def get_config_form( self ): raise TypeError( "Abstract method" ) @@ -84,18 +94,23 @@ def get_runtime_inputs( self ): raise TypeError( "Abstract method" ) + def get_runtime_state( self ): raise TypeError( "Abstract method" ) + def encode_runtime_state( self, trans, state ): raise TypeError( "Abstract method" ) + def decode_runtime_state( self, trans, string ): raise TypeError( "Abstract method" ) + def update_runtime_state( self, trans, state, values ): raise TypeError( "Abstract method" ) def execute( self, trans, state ): raise TypeError( "Abstract method" ) + class InputDataModule( WorkflowModule ): type = "data_input" name = "Input dataset" @@ -105,12 +120,14 @@ module = Class( trans ) module.state = dict( name="Input Dataset" ) return module + @classmethod def from_dict( Class, trans, d, secure=True ): module = Class( trans ) state = from_json_string( d["tool_state"] ) module.state = dict( name=state.get( "name", "Input Dataset" ) ) return module + @classmethod def from_workflow_step( Class, trans, step ): module = Class( trans ) @@ -172,6 +189,7 @@ def execute( self, trans, state ): return None, dict( output=state.inputs['input']) + class ToolModule( WorkflowModule ): type = "tool" @@ -286,6 +304,7 @@ def get_data_inputs( self ): data_inputs = [] + def callback( input, value, prefixed_name, prefixed_label ): if isinstance( input, DataToolParameter ): data_inputs.append( dict( @@ -293,6 +312,7 @@ label=prefixed_label, multiple=input.multiple, extensions=input.extensions ) ) + visit_input_values( self.tool.inputs, self.state.inputs, callback ) return data_inputs @@ -338,6 +358,7 @@ make_runtime_key = incoming.get( 'make_runtime', None ) make_buildtime_key = incoming.get( 'make_buildtime', None ) + def item_callback( trans, key, input, value, error, old_value, context ): # Dummy value for Data parameters if isinstance( input, DataToolParameter ): @@ -354,6 +375,7 @@ return value, None else: return value, error + # Update state using incoming values errors = self.tool.update_state( self.trans, self.tool.inputs, self.state.inputs, incoming, item_callback=item_callback ) self.errors = errors or None @@ -370,6 +392,7 @@ input_connections_by_name = {} # Any connected input needs to have value DummyDataset (these # are not persisted so we need to do it every time) + def callback( input, value, prefixed_name, prefixed_label ): replacement = None if isinstance( input, DataToolParameter ): @@ -379,9 +402,12 @@ else: replacement = DummyDataset() return replacement + visit_input_values( self.tool.inputs, self.state.inputs, callback ) + class WorkflowModuleFactory( object ): + def __init__( self, module_types ): self.module_types = module_types Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.

1 0