13 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/0a6e7085ce60/ Changeset: 0a6e7085ce60 User: dan Date: 2015-02-26 22:16:55+00:00 Summary: Allow TabularToolDataTable to be loaded from a url instead of a path. Affected #: 1 file diff -r 71596d6f10285ebb23d2a257eda3b70cb04ff655 -r 0a6e7085ce60c53e4a97aedd2e0561177b6f913a lib/galaxy/tools/data/__init__.py --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -14,6 +14,8 @@ import hashlib from glob import glob +from tempfile import NamedTemporaryFile +from urllib2 import urlopen from galaxy import util from galaxy.util.odict import odict @@ -279,7 +281,22 @@ repo_info = None # Read every file for file_element in config_element.findall( 'file' ): - filename = file_path = expand_here_template( file_element.get( 'path', None ), here=self.here ) + tmp_file = None + filename = file_element.get( 'path', None ) + if filename is None: + # Handle URLs as files + filename = file_element.get( 'url', None ) + if filename: + tmp_file = NamedTemporaryFile( prefix='TTDT_URL_%s-' % self.name ) + try: + tmp_file.write( urlopen( filename ).read() ) + except Exception, e: + log.error( 'Error loading Data Table URL "%s": %s', filename, e ) + continue + log.debug( 'Loading Data Table URL "%s" as filename "%s".', filename, tmp_file.name ) + filename = tmp_file.name + tmp_file.flush() + filename = file_path = expand_here_template( filename, here=self.here ) found = False if file_path is None: log.debug( "Encountered a file element (%s) that does not contain a path value when loading tool data table '%s'.", util.xml_to_string( file_element ), self.name ) @@ -325,6 +342,9 @@ config_element=config_element, tool_shed_repository=repo_info, errors=errors ) else: log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, self.filenames.keys() ) + # Remove URL tmp file + if tmp_file is not None: + tmp_file.close() def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ): assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns ) https://bitbucket.org/galaxy/galaxy-central/commits/b6fe31e55f48/ Changeset: b6fe31e55f48 User: dannon Date: 2015-02-27 00:59:26+00:00 Summary: Fix server_starttime being in app for reports, toolshed. I will be refactoring this shortly to have a base UniverseApplication from which things inherit. Affected #: 2 files diff -r 0a6e7085ce60c53e4a97aedd2e0561177b6f913a -r b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b lib/galaxy/webapps/reports/app.py --- a/lib/galaxy/webapps/reports/app.py +++ b/lib/galaxy/webapps/reports/app.py @@ -1,5 +1,7 @@ +import config import sys -import config +import time + import galaxy.model from galaxy.web import security @@ -26,6 +28,8 @@ self.targets_mysql = 'mysql' in self.config.database_connection # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) + # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. + self.server_starttime = int(time.time()) def shutdown( self ): pass diff -r 0a6e7085ce60c53e4a97aedd2e0561177b6f913a -r b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b lib/galaxy/webapps/tool_shed/app.py --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -1,17 +1,18 @@ import config import sys +import time +import galaxy.datatypes.registry +import galaxy.quota +import galaxy.tools.data +import galaxy.webapps.tool_shed.model from galaxy import tools -import galaxy.tools.data -import galaxy.quota -import galaxy.datatypes.registry -import galaxy.webapps.tool_shed.model +from galaxy.managers.tags import CommunityTagManager from galaxy.openid.providers import OpenIDProviders from galaxy.util.dbkeys import GenomeBuilds from galaxy.web import security -from galaxy.managers.tags import CommunityTagManager -from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager import tool_shed.repository_registry import tool_shed.repository_types.registry +from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager class UniverseApplication( object ): @@ -70,6 +71,8 @@ self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry( self ) + # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. + self.server_starttime = int(time.time()) print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config def shutdown( self ): https://bitbucket.org/galaxy/galaxy-central/commits/eed154f337fc/ Changeset: eed154f337fc User: dannon Date: 2015-02-27 01:13:48+00:00 Summary: Fix reports webapp under default database broken in ba825fc5 Affected #: 1 file diff -r b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b -r eed154f337fc090168956bcbec92491d0a5d75c9 lib/galaxy/webapps/reports/app.py --- a/lib/galaxy/webapps/reports/app.py +++ b/lib/galaxy/webapps/reports/app.py @@ -25,7 +25,10 @@ db_url, self.config.database_engine_options, create_tables=True ) - self.targets_mysql = 'mysql' in self.config.database_connection + if not self.config.database_connection: + self.targets_mysql = False + else: + self.targets_mysql = 'mysql' in self.config.database_connection # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. https://bitbucket.org/galaxy/galaxy-central/commits/4e30957ab121/ Changeset: 4e30957ab121 User: jmchilton Date: 2015-02-27 03:36:14+00:00 Summary: Fix incorrect usage of __all__ not containing strings... ...for plugin_config.py plugins. Affected #: 12 files diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/core.py --- a/lib/galaxy/jobs/metrics/instrumenters/core.py +++ b/lib/galaxy/jobs/metrics/instrumenters/core.py @@ -82,4 +82,4 @@ pass return value -__all__ = [ CorePlugin ] +__all__ = [ 'CorePlugin' ] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py --- a/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py +++ b/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py @@ -59,4 +59,4 @@ def __instrument_cpuinfo_path( self, job_directory ): return self._instrument_file_path( job_directory, "cpuinfo" ) -__all__ = [ CpuInfoPlugin ] +__all__ = [ 'CpuInfoPlugin' ] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/env.py --- a/lib/galaxy/jobs/metrics/instrumenters/env.py +++ b/lib/galaxy/jobs/metrics/instrumenters/env.py @@ -68,4 +68,4 @@ def __env_file( self, job_directory ): return self._instrument_file_path( job_directory, "vars" ) -__all__ = [ EnvPlugin ] +__all__ = [ 'EnvPlugin' ] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/meminfo.py --- a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py +++ b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py @@ -56,4 +56,4 @@ def __instrument_meminfo_path( self, job_directory ): return self._instrument_file_path( job_directory, "meminfo" ) -__all__ = [ MemInfoPlugin ] +__all__ = [ 'MemInfoPlugin' ] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/uname.py --- a/lib/galaxy/jobs/metrics/instrumenters/uname.py +++ b/lib/galaxy/jobs/metrics/instrumenters/uname.py @@ -31,4 +31,4 @@ return self._instrument_file_path( job_directory, "uname" ) -__all__ = [ UnamePlugin ] +__all__ = [ 'UnamePlugin' ] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py @@ -147,4 +147,4 @@ return base -__all__ = [HomebrewToolShedDependencyResolver] +__all__ = ['HomebrewToolShedDependencyResolver'] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/galaxy_packages.py --- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py +++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py @@ -71,4 +71,4 @@ commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % ( base_path, self.script ) return commands -__all__ = [GalaxyPackageDependencyResolver, GalaxyPackageDependency] +__all__ = ['GalaxyPackageDependencyResolver', 'GalaxyPackageDependency'] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/homebrew.py --- a/lib/galaxy/tools/deps/resolvers/homebrew.py +++ b/lib/galaxy/tools/deps/resolvers/homebrew.py @@ -53,4 +53,4 @@ return str( value ).lower() == "true" -__all__ = [HomebrewDependencyResolver] +__all__ = ['HomebrewDependencyResolver'] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/modules.py --- a/lib/galaxy/tools/deps/resolvers/modules.py +++ b/lib/galaxy/tools/deps/resolvers/modules.py @@ -153,4 +153,4 @@ def _string_as_bool( value ): return str( value ).lower() == "true" -__all__ = [ModuleDependencyResolver] +__all__ = ['ModuleDependencyResolver'] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/tool_shed_packages.py --- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py +++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py @@ -56,4 +56,4 @@ return INDETERMINATE_DEPENDENCY -__all__ = [ToolShedPackageDependencyResolver] +__all__ = ['ToolShedPackageDependencyResolver'] diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/util/plugin_config.py --- a/lib/galaxy/util/plugin_config.py +++ b/lib/galaxy/util/plugin_config.py @@ -25,6 +25,10 @@ # FIXME: this is not how one is suppose to use __all__ why did you do # this past John? for clazz in getattr( plugin_module, "__all__", [] ): + try: + clazz = getattr( plugin_module, clazz ) + except TypeError: + clazz = clazz plugin_type = getattr( clazz, plugin_type_identifier, None ) if plugin_type: plugin_dict[ plugin_type ] = clazz diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r 4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/workflow/schedulers/core.py --- a/lib/galaxy/workflow/schedulers/core.py +++ b/lib/galaxy/workflow/schedulers/core.py @@ -43,4 +43,4 @@ workflow_invocation=workflow_invocation, ) -__all__ = [ CoreWorkflowSchedulingPlugin ] +__all__ = [ 'CoreWorkflowSchedulingPlugin' ] https://bitbucket.org/galaxy/galaxy-central/commits/e8c37fd94209/ Changeset: e8c37fd94209 User: jmchilton Date: 2015-02-27 05:29:07+00:00 Summary: Update pulsar client and related modules. Affected #: 24 files diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/cli.py --- a/lib/galaxy/jobs/metrics/collectl/cli.py +++ b/lib/galaxy/jobs/metrics/collectl/cli.py @@ -12,7 +12,8 @@ class CollectlCli( object ): - """ Abstraction over (some of) the command-line arguments of collectl. + """ + Abstraction over (some of) the command-line arguments of collectl. Ideally this will be useful for building up command line arguments for remote execution as well as runnning directly on local host. @@ -20,33 +21,43 @@ collectl CLI - logic more directly related to the Galaxy job metric plugin plugin should be placed in other modules. - Keyword Arguments: - collectl_path: Path to collectl executable (defaults to collectl - i.e. - search the PATH). + **Keyword Arguments:** - playback_path (defaults to None): If this is None collectl will run in - record mode, else it will playback specified file. + ``collectl_path`` + Path to collectl executable (defaults to collectl - i.e. + search the PATH). - Playback Mode Options: - - sep : Separator used in playback mode (set to 9 to produce tsv) - (defaults to None). + ``playback_path`` (defaults to ``None``) + If this is ``None``, collectl will run in + record mode, else it will playback specified file. - Record Mode Options (some of these may work in playback mode also): + **Playback Mode Options:** - destination_path: Location of path files to write to (defaults to None - and collectl will just use cwd). Really this is just to prefix - - collectl will append hostname and datetime to file. - interval: Setup polling interval (secs) for most subsystems (defaults - to None and when unspecified collectl will use default of 1 second). - interval2: Setup polling interval (secs) for process information - (defaults to None and when unspecified collectl will use default to - 60 seconds). - interval3: Setup polling interval (secs) for environment information - (defaults to None and when unspecified collectl will use default to - 300 seconds). - procfilt: Optional argument to procfilt. (defaults to None). - flush : Optional flush interval (defaults to None). + ``sep`` + Separator used in playback mode (set to 9 to produce tsv) + (defaults to None). + + **Record Mode Options** (some of these may work in playback mode also) + + ``destination_path`` + Location of path files to write to (defaults to None + and collectl will just use cwd). Really this is just to prefix - + collectl will append hostname and datetime to file. + ``interval`` + Setup polling interval (secs) for most subsystems (defaults + to None and when unspecified collectl will use default of 1 second). + ``interval2`` + Setup polling interval (secs) for process information + (defaults to None and when unspecified collectl will use default to + 60 seconds). + ``interval3`` + Setup polling interval (secs) for environment information + (defaults to None and when unspecified collectl will use default to + 300 seconds). + ``procfilt`` + Optional argument to procfilt. (defaults to None). + ``flush`` + Optional flush interval (defaults to None). """ def __init__( self, **kwargs ): @@ -125,4 +136,4 @@ if return_code: raise Exception( "Problem running collectl command." ) -__all__ = [ CollectlCli ] +__all__ = [ 'CollectlCli' ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/processes.py --- a/lib/galaxy/jobs/metrics/collectl/processes.py +++ b/lib/galaxy/jobs/metrics/collectl/processes.py @@ -26,7 +26,7 @@ # # Process data dumped one row per process per interval. -#http://collectl.sourceforge.net/Data-detail.html +# http://collectl.sourceforge.net/Data-detail.html PROCESS_COLUMNS = [ "#Date", # Date of interval - e.g. 20140322 "Time", # Time of interval - 12:18:58 @@ -36,7 +36,7 @@ "PPID", # Parent PID of process. "THRD", # Thread??? "S", # Process state - S - Sleeping, D - Uninterruptable Sleep, R - Running, Z - Zombie or T - Stopped/Traced - ## Memory options - http://ewx.livejournal.com/579283.html + # Memory options - http://ewx.livejournal.com/579283.html "VmSize", "VmLck", "VmRSS", @@ -93,7 +93,7 @@ statistics = DEFAULT_STATISTICS statistics = util.listify( statistics ) - statistics = map( __tuplize_statistic, statistics ) + statistics = map( _tuplize_statistic, statistics ) # Check for validity... for statistic in statistics: if statistic[ 0 ] not in STATISTIC_TYPES: @@ -109,10 +109,10 @@ with tempfile.NamedTemporaryFile( ) as tmp_tsv: collectl_playback_cli.run( stdout=tmp_tsv ) with open( tmp_tsv.name, "r" ) as tsv_file: - return __read_process_statistics( tsv_file, pid, statistics ) + return _read_process_statistics( tsv_file, pid, statistics ) -def __read_process_statistics( tsv_file, pid, statistics ): +def _read_process_statistics( tsv_file, pid, statistics ): process_summarizer = CollectlProcessSummarizer( pid, statistics ) current_interval = None @@ -167,7 +167,7 @@ to_num = float else: to_num = long - + interval_stat = sum( to_num( r[ column_index ] ) for r in rows ) self.tree_statistics[ column_name ].track( interval_stat ) @@ -228,7 +228,7 @@ ability to filter out just rows corresponding to the process tree corresponding to a given pid. """ - + def __init__( self ): self.rows = [] @@ -242,11 +242,11 @@ self.rows.append( row ) -def __tuplize_statistic( statistic ): +def _tuplize_statistic( statistic ): if not isinstance( statistic, tuple ): statistic_split = statistic.split( "_", 1 ) statistic = ( statistic_split[ 0 ].lower(), statistic_split[ 1 ] ) return statistic -__all__ = [ generate_process_statistics ] +__all__ = [ 'generate_process_statistics' ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/subsystems.py --- a/lib/galaxy/jobs/metrics/collectl/subsystems.py +++ b/lib/galaxy/jobs/metrics/collectl/subsystems.py @@ -69,4 +69,4 @@ """ return SUBSYSTEM_DICT[ name ] -__all__ = [ get_subsystem ] +__all__ = [ 'get_subsystem' ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/instrumenters/collectl.py --- a/lib/galaxy/jobs/metrics/instrumenters/collectl.py +++ b/lib/galaxy/jobs/metrics/instrumenters/collectl.py @@ -211,4 +211,4 @@ return "" -__all__ = [ CollectlPlugin ] +__all__ = [ 'CollectlPlugin' ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/__init__.py --- a/lib/galaxy/jobs/runners/util/__init__.py +++ b/lib/galaxy/jobs/runners/util/__init__.py @@ -7,4 +7,4 @@ from .kill import kill_pid -__all__ = [kill_pid, Bunch] +__all__ = ['kill_pid', 'Bunch'] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/__init__.py --- a/lib/galaxy/jobs/runners/util/cli/__init__.py +++ b/lib/galaxy/jobs/runners/util/cli/__init__.py @@ -6,6 +6,9 @@ DEFAULT_SHELL_PLUGIN = 'LocalShell' +ERROR_MESSAGE_NO_JOB_PLUGIN = "No job plugin parameter found, cannot create CLI job interface" +ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN = "Failed to find job_plugin of type %s, available types include %s" + class CliInterface(object): """ @@ -53,8 +56,14 @@ return shell def get_job_interface(self, job_params): - job_plugin = job_params['plugin'] - job_interface = self.cli_job_interfaces[job_plugin](**job_params) + job_plugin = job_params.get('plugin', None) + if not job_plugin: + raise ValueError(ERROR_MESSAGE_NO_JOB_PLUGIN) + job_plugin_class = self.cli_job_interfaces.get(job_plugin, None) + if not job_plugin_class: + raise ValueError(ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN % (job_plugin, self.cli_job_interfaces.keys())) + job_interface = job_plugin_class(**job_params) + return job_interface diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/factory.py --- a/lib/galaxy/jobs/runners/util/cli/factory.py +++ b/lib/galaxy/jobs/runners/util/cli/factory.py @@ -12,7 +12,11 @@ code_dir = '.' +def build_cli_interface(): + return CliInterface(code_dir=code_dir) + + def get_shell(params): - cli_interface = CliInterface(code_dir=code_dir) + cli_interface = build_cli_interface() shell_params, _ = split_params(params) return cli_interface.get_shell_plugin(shell_params) diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/job/slurm.py --- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py +++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py @@ -1,6 +1,10 @@ # A simple CLI runner for slurm that can be used when running Galaxy from a # non-submit host and using a Slurm cluster. +from ..job import BaseJobExec + +from logging import getLogger + try: from galaxy.model import Job job_states = Job.states @@ -9,11 +13,6 @@ from galaxy.util import enum job_states = enum(RUNNING='running', OK='complete', QUEUED='queued', ERROR="failed") -from ..job import BaseJobExec - -__all__ = ('Slurm',) - -from logging import getLogger log = getLogger(__name__) argmap = { @@ -94,3 +93,6 @@ }.get(state) except KeyError: raise KeyError("Failed to map slurm status code [%s] to job state." % state) + + +__all__ = ('Slurm',) diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/job/torque.py --- a/lib/galaxy/jobs/runners/util/cli/job/torque.py +++ b/lib/galaxy/jobs/runners/util/cli/job/torque.py @@ -1,3 +1,4 @@ +from logging import getLogger try: import xml.etree.cElementTree as et except: @@ -13,10 +14,10 @@ from ..job import BaseJobExec -__all__ = ('Torque',) +log = getLogger(__name__) -from logging import getLogger -log = getLogger(__name__) +ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to Torque CLI plugin: %s' + argmap = {'destination': '-q', 'Execution_Time': '-a', @@ -58,8 +59,8 @@ if not k.startswith('-'): k = argmap[k] pbsargs[k] = v - except: - log.warning('Unrecognized long argument passed to Torque CLI plugin: %s' % k) + except KeyError: + log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k) template_pbsargs = '' for k, v in pbsargs.items(): template_pbsargs += '#PBS %s %s\n' % (k, v) @@ -118,3 +119,6 @@ }.get(state) except KeyError: raise KeyError("Failed to map torque status code [%s] to job state." % state) + + +__all__ = ('Torque',) diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/shell/local.py --- a/lib/galaxy/jobs/runners/util/cli/shell/local.py +++ b/lib/galaxy/jobs/runners/util/cli/shell/local.py @@ -24,7 +24,7 @@ True True - >>> exec_result = exec_python("import time; time.sleep(90)", timeout=3, timeout_check_interval=1) + >>> exec_result = exec_python("import time; time.sleep(90)", timeout=1, timeout_check_interval=.1) True diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/__init__.py --- a/lib/pulsar/client/__init__.py +++ b/lib/pulsar/client/__init__.py @@ -1,6 +1,6 @@ """ pulsar client -====== +================= This module contains logic for interfacing with an external Pulsar server. @@ -50,13 +50,13 @@ from .path_mapper import PathMapper __all__ = [ - build_client_manager, - OutputNotFoundException, - url_to_destination_params, - finish_job, - submit_job, - ClientJobDescription, - PulsarOutputs, - ClientOutputs, - PathMapper, + 'build_client_manager', + 'OutputNotFoundException', + 'url_to_destination_params', + 'finish_job', + 'submit_job', + 'ClientJobDescription', + 'PulsarOutputs', + 'ClientOutputs', + 'PathMapper', ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/action_mapper.py --- a/lib/pulsar/client/action_mapper.py +++ b/lib/pulsar/client/action_mapper.py @@ -1,5 +1,6 @@ from json import load from os import makedirs +from os import unlink from os.path import exists from os.path import abspath from os.path import dirname @@ -16,6 +17,9 @@ from .util import unique_path_prefix from .transport import get_file from .transport import post_file +from .transport import rsync_get_file, scp_get_file +from .transport import rsync_post_file, scp_post_file +import tempfile DEFAULT_MAPPED_ACTION = 'transfer' # Not really clear to me what this should be, exception? @@ -130,6 +134,7 @@ if config is None: config = self.__client_to_config(client) self.default_action = config.get("default_action", "transfer") + self.ssh_key = config.get("ssh_key", None) self.mappers = mappers_from_dicts(config.get("paths", [])) self.files_endpoint = config.get("files_endpoint", None) @@ -155,6 +160,7 @@ return dict( default_action=self.default_action, files_endpoint=self.files_endpoint, + ssh_key=self.ssh_key, paths=map(lambda m: m.to_dict(), self.mappers) ) @@ -166,6 +172,8 @@ config = dict() config["default_action"] = client.default_file_action config["files_endpoint"] = client.files_endpoint + if hasattr(client, 'ssh_key'): + config["ssh_key"] = client.ssh_key return config def __load_action_config(self, path): @@ -209,6 +217,9 @@ # TODO: URL encode path. url = "%s&path=%s&file_type=%s" % (url_base, action.path, file_type) action.url = url + elif action.action_type in ["remote_rsync_transfer", "remote_scp_transfer"]: + # Required, so no check for presence + action.ssh_key = self.ssh_key REQUIRED_ACTION_KWD = object() @@ -375,6 +386,95 @@ post_file(self.url, pulsar_path) +class PubkeyAuthenticatedTransferAction(BaseAction): + """Base class for file transfers requiring an SSH public/private key + """ + action_spec = dict( + ssh_user=REQUIRED_ACTION_KWD, + ssh_host=REQUIRED_ACTION_KWD, + ssh_port=REQUIRED_ACTION_KWD, + ) + action_type = "remote_pubkey_transfer" + staging = STAGING_ACTION_REMOTE + ssh_key = None + + def __init__(self, path, file_lister=None, url=None, ssh_user=None, + ssh_host=None, ssh_port=None, ssh_key=None): + super(PubkeyAuthenticatedTransferAction, self).__init__(path, file_lister=file_lister) + self.url = url + self.ssh_user = ssh_user + self.ssh_host = ssh_host + self.ssh_port = ssh_port + self.ssh_key = ssh_key + + def to_dict(self): + return dict(path=self.path, action_type=self.action_type, url=self.url, + ssh_user=self.ssh_user, ssh_host=self.ssh_host, + ssh_port=self.ssh_port) + + def serialize_ssh_key(self): + f = tempfile.NamedTemporaryFile(delete=False) + if self.ssh_key is not None: + f.write(self.ssh_key) + else: + raise Exception("SSH_KEY not available") + return f.name + + def cleanup_ssh_key(self, keyfile): + if exists(keyfile): + unlink(keyfile) + + +class RsyncTransferAction(PubkeyAuthenticatedTransferAction): + action_type = "remote_rsync_transfer" + + @classmethod + def from_dict(cls, action_dict): + return RsyncTransferAction(path=action_dict["path"], + url=action_dict["url"], + ssh_user=action_dict["ssh_user"], + ssh_host=action_dict["ssh_host"], + ssh_port=action_dict["ssh_port"], + ssh_key=action_dict["ssh_key"]) + + def write_to_path(self, path): + key_file = self.serialize_ssh_key() + rsync_get_file(self.path, path, self.ssh_user, self.ssh_host, + self.ssh_port, key_file) + self.cleanup_ssh_key(key_file) + + def write_from_path(self, pulsar_path): + key_file = self.serialize_ssh_key() + rsync_post_file(pulsar_path, self.path, self.ssh_user, + self.ssh_host, self.ssh_port, key_file) + self.cleanup_ssh_key(key_file) + + +class ScpTransferAction(PubkeyAuthenticatedTransferAction): + action_type = "remote_scp_transfer" + + @classmethod + def from_dict(cls, action_dict): + return ScpTransferAction(path=action_dict["path"], + url=action_dict["url"], + ssh_user=action_dict["ssh_user"], + ssh_host=action_dict["ssh_host"], + ssh_port=action_dict["ssh_port"], + ssh_key=action_dict["ssh_key"]) + + def write_to_path(self, path): + key_file = self.serialize_ssh_key() + scp_get_file(self.path, path, self.ssh_user, self.ssh_host, + self.ssh_port, key_file) + self.cleanup_ssh_key(key_file) + + def write_from_path(self, pulsar_path): + key_file = self.serialize_ssh_key() + scp_post_file(pulsar_path, self.path, self.ssh_user, self.ssh_host, + self.ssh_port, key_file) + self.cleanup_ssh_key(key_file) + + class MessageAction(object): """ Sort of pseudo action describing "files" store in memory and transferred via message (HTTP, Python-call, MQ, etc...) @@ -408,7 +508,7 @@ open(path, "w").write(self.contents) -DICTIFIABLE_ACTION_CLASSES = [RemoteCopyAction, RemoteTransferAction, MessageAction] +DICTIFIABLE_ACTION_CLASSES = [RemoteCopyAction, RemoteTransferAction, MessageAction, RsyncTransferAction, ScpTransferAction] def from_dict(action_dict): @@ -517,10 +617,10 @@ def mappers_from_dicts(mapper_def_list): - return map(lambda m: __mappper_from_dict(m), mapper_def_list) + return map(lambda m: _mappper_from_dict(m), mapper_def_list) -def __mappper_from_dict(mapper_dict): +def _mappper_from_dict(mapper_dict): map_type = mapper_dict.get('match_type', DEFAULT_PATH_MAPPER_TYPE) return MAPPER_CLASS_DICT[map_type](mapper_dict) @@ -554,14 +654,16 @@ CopyAction, RemoteCopyAction, RemoteTransferAction, + RsyncTransferAction, + ScpTransferAction, ] actions = dict([(clazz.action_type, clazz) for clazz in ACTION_CLASSES]) __all__ = [ - FileActionMapper, - path_type, - from_dict, - MessageAction, - RemoteTransferAction, # For testing + 'FileActionMapper', + 'path_type', + 'from_dict', + 'MessageAction', + 'RemoteTransferAction', # For testing ] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/amqp_exchange.py --- a/lib/pulsar/client/amqp_exchange.py +++ b/lib/pulsar/client/amqp_exchange.py @@ -1,13 +1,16 @@ +import copy +import uuid +import socket +import logging +import threading +from time import sleep + try: import kombu from kombu import pools except ImportError: kombu = None -import socket -import logging -import threading -from time import sleep log = logging.getLogger(__name__) @@ -101,24 +104,45 @@ log.debug('AMQP heartbeat thread exiting') def publish(self, name, payload): + # Consider optionally disabling if throughput becomes main concern. + transaction_uuid = uuid.uuid1() key = self.__queue_name(name) - log.debug("Begin publishing to key %s" % key) + publish_log_prefix = self.__publish_log_prefex(transaction_uuid) + log.debug("%sBegin publishing to key %s", publish_log_prefix, key) with self.connection(self.__url) as connection: with pools.producers[connection].acquire() as producer: - log.debug("Have producer for publishing to key %s" % key) + log.debug("%sHave producer for publishing to key %s", publish_log_prefix, key) + publish_kwds = self.__prepare_publish_kwds(publish_log_prefix) producer.publish( payload, serializer='json', exchange=self.__exchange, declare=[self.__exchange], routing_key=key, - **self.__publish_kwds + **publish_kwds ) - log.debug("Published to key %s" % key) + log.debug("%sPublished to key %s", publish_log_prefix, key) - def __publish_errback(self, exc, interval): - log.error("Connection error while publishing: %r", exc, exc_info=1) - log.info("Retrying in %s seconds", interval) + def __prepare_publish_kwds(self, publish_log_prefix): + if "retry_policy" in self.__publish_kwds: + publish_kwds = copy.deepcopy(self.__publish_kwds) + + def errback(exc, interval): + return self.__publish_errback(exc, interval, publish_log_prefix) + publish_kwds["retry_policy"]["errback"] = errback + else: + publish_kwds = self.__publish_kwds + return publish_kwds + + def __publish_errback(self, exc, interval, publish_log_prefix=""): + log.error("%sConnection error while publishing: %r", publish_log_prefix, exc, exc_info=1) + log.info("%sRetrying in %s seconds", publish_log_prefix, interval) + + def __publish_log_prefex(self, transaction_uuid=None): + prefix = "" + if transaction_uuid: + prefix = "[publish:%s] " % str(transaction_uuid) + return prefix def connection(self, connection_string, **kwargs): if "ssl" not in kwargs: diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/client.py --- a/lib/pulsar/client/client.py +++ b/lib/pulsar/client/client.py @@ -41,6 +41,12 @@ ) else: job_directory = None + + if "ssh_key" in (destination_params or {}): + self.ssh_key = destination_params["ssh_key"] + else: + self.ssh_key = None + self.env = destination_params.get("env", []) self.files_endpoint = destination_params.get("files_endpoint", None) self.job_directory = job_directory @@ -271,6 +277,7 @@ launch_params['env'] = env if remote_staging: launch_params['remote_staging'] = remote_staging + launch_params['remote_staging']['ssh_key'] = self.ssh_key if job_config and self.setup_handler.local: # Setup not yet called, job properties were inferred from # destination arguments. Hence, must have Pulsar setup job diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/manager.py --- a/lib/pulsar/client/manager.py +++ b/lib/pulsar/client/manager.py @@ -220,4 +220,8 @@ int_val = int(val) return int_val -__all__ = [ClientManager, ObjectStoreClientManager, HttpPulsarInterface] +__all__ = [ + 'ClientManager', + 'ObjectStoreClientManager', + 'HttpPulsarInterface' +] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/path_mapper.py --- a/lib/pulsar/client/path_mapper.py +++ b/lib/pulsar/client/path_mapper.py @@ -95,4 +95,4 @@ message = "PathMapper cannot handle path type %s" % dataset_path_type raise Exception(message) -__all__ = [PathMapper] +__all__ = ['PathMapper'] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/setup_handler.py --- a/lib/pulsar/client/setup_handler.py +++ b/lib/pulsar/client/setup_handler.py @@ -82,6 +82,7 @@ unstructured_files_directory = job_directory.unstructured_files_directory() sep = system_properties.get("sep", os.sep) job_config = { + "job_directory": job_directory.path, "working_directory": working_directory, "outputs_directory": outputs_directory, "configs_directory": configs_directory, @@ -100,4 +101,4 @@ return job_config -__all__ = [build_job_config, build] +__all__ = ['build_job_config', 'build'] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/staging/down.py --- a/lib/pulsar/client/staging/down.py +++ b/lib/pulsar/client/staging/down.py @@ -152,4 +152,4 @@ except Exception: log.warn("Failed to cleanup remote Pulsar job") -__all__ = [finish_job] +__all__ = ['finish_job'] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/staging/up.py --- a/lib/pulsar/client/staging/up.py +++ b/lib/pulsar/client/staging/up.py @@ -349,14 +349,18 @@ local_action = action.staging_action_local if local_action: response = self.client.put_file(path, type, name=name, contents=contents) - get_path = lambda: response['path'] + + def get_path(): + return response['path'] else: job_directory = self.client.job_directory assert job_directory, "job directory required for action %s" % action if not name: name = basename(path) self.__add_remote_staging_input(action, name, type) - get_path = lambda: job_directory.calculate_path(name, type) + + def get_path(): + return job_directory.calculate_path(name, type) register = self.rewrite_paths or type == 'tool' # Even if inputs not rewritten, tool must be. if register: self.register_rewrite(path, get_path(), type, force=True) @@ -417,4 +421,4 @@ input.close() -__all__ = [submit_job] +__all__ = ['submit_job'] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/__init__.py --- a/lib/pulsar/client/transport/__init__.py +++ b/lib/pulsar/client/transport/__init__.py @@ -2,26 +2,8 @@ from .curl import PycurlTransport import os - -def get_transport(transport_type=None, os_module=os): - transport_type = __get_transport_type(transport_type, os_module) - if transport_type == 'urllib': - transport = Urllib2Transport() - else: - transport = PycurlTransport() - return transport - - -def __get_transport_type(transport_type, os_module): - if not transport_type: - use_curl = os_module.getenv('PULSAR_CURL_TRANSPORT', "0") - # If PULSAR_CURL_TRANSPORT is unset or set to 0, use default, - # else use curl. - if use_curl.isdigit() and not int(use_curl): - transport_type = 'urllib' - else: - transport_type = 'curl' - return transport_type +from .ssh import rsync_get_file, scp_get_file +from .ssh import rsync_post_file, scp_post_file from .curl import curl_available from .requests import requests_multipart_post_available @@ -35,4 +17,33 @@ from .poster import get_file from .poster import post_file -__all__ = [get_transport, get_file, post_file] + +def get_transport(transport_type=None, os_module=os): + transport_type = _get_transport_type(transport_type, os_module) + if transport_type == 'urllib': + transport = Urllib2Transport() + else: + transport = PycurlTransport() + return transport + + +def _get_transport_type(transport_type, os_module): + if not transport_type: + use_curl = os_module.getenv('PULSAR_CURL_TRANSPORT', "0") + # If PULSAR_CURL_TRANSPORT is unset or set to 0, use default, + # else use curl. + if use_curl.isdigit() and not int(use_curl): + transport_type = 'urllib' + else: + transport_type = 'curl' + return transport_type + +__all__ = [ + 'get_transport', + 'get_file', + 'post_file', + 'rsync_get_file', + 'rsync_post_file', + 'scp_get_file', + 'scp_post_file' +] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/curl.py --- a/lib/pulsar/client/transport/curl.py +++ b/lib/pulsar/client/transport/curl.py @@ -2,32 +2,37 @@ from cStringIO import StringIO except ImportError: from io import StringIO -curl_available = True + try: - from pycurl import Curl + from pycurl import Curl, HTTP_CODE + curl_available = True except ImportError: curl_available = False -from os.path import getsize + +import os.path PYCURL_UNAVAILABLE_MESSAGE = \ "You are attempting to use the Pycurl version of the Pulsar client but pycurl is unavailable." +NO_SUCH_FILE_MESSAGE = "Attempt to post file %s to URL %s, but file does not exist." +POST_FAILED_MESSAGE = "Failed to post_file properly for url %s, remote server returned status code of %s." +GET_FAILED_MESSAGE = "Failed to get_file properly for url %s, remote server returned status code of %s." + class PycurlTransport(object): def execute(self, url, method=None, data=None, input_path=None, output_path=None): buf = _open_output(output_path) try: - c = _new_curl_object() - c.setopt(c.URL, url.encode('ascii')) + c = _new_curl_object_for_url(url) c.setopt(c.WRITEFUNCTION, buf.write) if method: c.setopt(c.CUSTOMREQUEST, method) if input_path: c.setopt(c.UPLOAD, 1) c.setopt(c.READFUNCTION, open(input_path, 'rb').read) - filesize = getsize(input_path) + filesize = os.path.getsize(input_path) c.setopt(c.INFILESIZE, filesize) if data: c.setopt(c.POST, 1) @@ -42,19 +47,30 @@ def post_file(url, path): - c = _new_curl_object() - c.setopt(c.URL, url.encode('ascii')) + if not os.path.exists(path): + # pycurl doesn't always produce a great exception for this, + # wrap it in a better one. + message = NO_SUCH_FILE_MESSAGE % (path, url) + raise Exception(message) + c = _new_curl_object_for_url(url) c.setopt(c.HTTPPOST, [("file", (c.FORM_FILE, path.encode('ascii')))]) c.perform() + status_code = c.getinfo(HTTP_CODE) + if int(status_code) != 200: + message = POST_FAILED_MESSAGE % (url, status_code) + raise Exception(message) def get_file(url, path): buf = _open_output(path) try: - c = _new_curl_object() - c.setopt(c.URL, url.encode('ascii')) + c = _new_curl_object_for_url(url) c.setopt(c.WRITEFUNCTION, buf.write) c.perform() + status_code = c.getinfo(HTTP_CODE) + if int(status_code) != 200: + message = GET_FAILED_MESSAGE % (url, status_code) + raise Exception(message) finally: buf.close() @@ -63,10 +79,20 @@ return open(output_path, 'wb') if output_path else StringIO() +def _new_curl_object_for_url(url): + c = _new_curl_object() + c.setopt(c.URL, url.encode('ascii')) + return c + + def _new_curl_object(): try: return Curl() except NameError: raise ImportError(PYCURL_UNAVAILABLE_MESSAGE) -___all__ = [PycurlTransport, post_file, get_file] +__all__ = [ + 'PycurlTransport', + 'post_file', + 'get_file' +] diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/poster.py --- a/lib/pulsar/client/transport/poster.py +++ b/lib/pulsar/client/transport/poster.py @@ -1,4 +1,7 @@ from __future__ import absolute_import + +import logging + try: from urllib2 import urlopen except ImportError: @@ -20,7 +23,6 @@ POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is unavailable. Please install poster." -import logging log = logging.getLogger(__name__) diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/requests.py --- a/lib/pulsar/client/transport/requests.py +++ b/lib/pulsar/client/transport/requests.py @@ -1,4 +1,7 @@ from __future__ import absolute_import + +import logging + try: from galaxy import eggs eggs.require("requests") @@ -9,18 +12,18 @@ import requests except ImportError: requests = None -requests_multipart_post_available = False + try: import requests_toolbelt requests_multipart_post_available = True except ImportError: + requests_multipart_post_available = False requests_toolbelt = None REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is unavailable. Please install requests." REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt module - but it is unavailable. Please install requests_toolbelt." -import logging log = logging.getLogger(__name__) diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/ssh.py --- /dev/null +++ b/lib/pulsar/client/transport/ssh.py @@ -0,0 +1,64 @@ +import subprocess +SSH_OPTIONS = ('-o', 'StrictHostKeyChecking=no', '-o', 'PreferredAuthentications=publickey', '-o', 'PubkeyAuthentication=yes') + + +def rsync_get_file(uri_from, uri_to, user, host, port, key): + cmd = [ + 'rsync', + '-e', + 'ssh -i %s -p %s %s' % (key, port, ' '.join(SSH_OPTIONS)), + '%s@%s:%s' % (user, host, uri_from), + uri_to, + ] + exit_code = subprocess.check_call(cmd) + if exit_code != 0: + raise Exception("Rsync exited with code %s" % exit_code) + + +def rsync_post_file(uri_from, uri_to, user, host, port, key): + cmd = [ + 'rsync', + '-e', + 'ssh -i %s -p %s %s' % (key, port, ' '.join(SSH_OPTIONS)), + uri_from, + '%s@%s:%s' % (user, host, uri_to), + ] + exit_code = subprocess.check_call(cmd) + if exit_code != 0: + raise Exception("Rsync exited with code %s" % exit_code) + + +def scp_get_file(uri_from, uri_to, user, host, port, key): + cmd = [ + 'scp', + '-P', str(port), + '-i', key, + SSH_OPTIONS, + '%s@%s:%s' % (user, host, uri_from), + uri_to, + ] + exit_code = subprocess.check_call(cmd) + if exit_code != 0: + raise Exception("scp exited with code %s" % exit_code) + + +def scp_post_file(uri_from, uri_to, user, host, port, key): + cmd = [ + 'scp', + '-P', str(port), + '-i', key, + SSH_OPTIONS, + uri_from, + '%s@%s:%s' % (user, host, uri_to), + ] + exit_code = subprocess.check_call(cmd) + if exit_code != 0: + raise Exception("scp exited with code %s" % exit_code) + + +___all__ = [ + 'rsync_post_file', + 'rsync_get_file', + 'scp_post_file', + 'scp_get_file' +] https://bitbucket.org/galaxy/galaxy-central/commits/e630e74d5571/ Changeset: e630e74d5571 User: guerler Date: 2015-02-27 06:30:48+00:00 Summary: ToolForm: Show initial validation errors on re-run Affected #: 3 files diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r e630e74d5571462111045b7db9b021427c7ab6ef client/galaxy/scripts/mvc/tools/tools-form.js --- a/client/galaxy/scripts/mvc/tools/tools-form.js +++ b/client/galaxy/scripts/mvc/tools/tools-form.js @@ -22,6 +22,7 @@ } }) } + options.initial_errors = options.job_id; ToolFormBase.prototype.initialize.call(this, options); }, diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r e630e74d5571462111045b7db9b021427c7ab6ef static/scripts/mvc/tools/tools-form.js --- a/static/scripts/mvc/tools/tools-form.js +++ b/static/scripts/mvc/tools/tools-form.js @@ -22,6 +22,7 @@ } }) } + options.initial_errors = options.job_id; ToolFormBase.prototype.initialize.call(this, options); }, diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r e630e74d5571462111045b7db9b021427c7ab6ef static/scripts/packed/mvc/tools/tools-form.js --- a/static/scripts/packed/mvc/tools/tools-form.js +++ b/static/scripts/packed/mvc/tools/tools-form.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var d=b.extend({initialize:function(g){var f=this;this.job_handler=new a(this);this.buttons={execute:new e.Button({icon:"fa-check",tooltip:"Execute: "+g.name,title:"Execute",cls:"btn btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};b.prototype.initialize.call(this,g)},_buildModel:function(){var f=this;var g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var i=top.location.href;var j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now you are using '"+f.options.name+"' version "+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught error.";f.modal.show({title:"Tool cannot be executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var h in g.inputs){var n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function m(r){for(var p in f.input_list){var q=f.field_list[p];var o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending current state (see below).");console.debug(g);var j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var u=f.field_list[p];if(u.update){var s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var r in t.options){var q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() - Received new model (see below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(i)}})}});return{View:d}}); \ No newline at end of file +define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var d=b.extend({initialize:function(g){var f=this;this.job_handler=new a(this);this.buttons={execute:new e.Button({icon:"fa-check",tooltip:"Execute: "+g.name,title:"Execute",cls:"btn btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var f=this;var g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var i=top.location.href;var j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now you are using '"+f.options.name+"' version "+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught error.";f.modal.show({title:"Tool cannot be executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var h in g.inputs){var n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function m(r){for(var p in f.input_list){var q=f.field_list[p];var o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending current state (see below).");console.debug(g);var j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var u=f.field_list[p];if(u.update){var s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var r in t.options){var q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() - Received new model (see below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(i)}})}});return{View:d}}); \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/eb6b7fb7a91c/ Changeset: eb6b7fb7a91c User: guerler Date: 2015-02-27 06:35:38+00:00 Summary: ToolForm/Collections: Re-introduce parameter expansion Affected #: 4 files diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 client/galaxy/scripts/mvc/tools/tools-form.js --- a/client/galaxy/scripts/mvc/tools/tools-form.js +++ b/client/galaxy/scripts/mvc/tools/tools-form.js @@ -110,17 +110,6 @@ inputs : current_state } - // patch data tool parameters - // TODO: This needs to be removed and handled in the api - for (var i in current_state.inputs) { - var dict = current_state.inputs[i]; - try { - if (dict && dict.values[0].src === 'hda') { - current_state.inputs[i] = self.content.get({id: dict.values[0].id, src: 'hda'}).id_uncoded; - } - } catch (err) {} - } - // activates/disables spinner for dynamic fields to indicate that they are currently being updated function wait(active) { for (var i in self.input_list) { diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 lib/galaxy/tools/__init__.py --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2366,7 +2366,7 @@ else: value, error = check_param(trans, input, default_value, context) except Exception, err: - log.error('Checking parameter failed. %s', str(err)) + log.error('Checking parameter %s failed. %s', input.name, str(err)) pass return [value, error] @@ -2482,6 +2482,11 @@ else: state[k] = jsonify(state[k]) + # expand incoming parameters (parameters might trigger multiple tool executions, + # here we select the first execution only in order to resolve dynamic parameters) + expanded_incomings, _ = expand_meta_parameters( trans, self, params.__dict__ ) + params.__dict__ = expanded_incomings[ 0 ] + # do param translation here, used by datasource tools if self.input_translator: self.input_translator.translate( params ) diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 static/scripts/mvc/tools/tools-form.js --- a/static/scripts/mvc/tools/tools-form.js +++ b/static/scripts/mvc/tools/tools-form.js @@ -110,17 +110,6 @@ inputs : current_state } - // patch data tool parameters - // TODO: This needs to be removed and handled in the api - for (var i in current_state.inputs) { - var dict = current_state.inputs[i]; - try { - if (dict && dict.values[0].src === 'hda') { - current_state.inputs[i] = self.content.get({id: dict.values[0].id, src: 'hda'}).id_uncoded; - } - } catch (err) {} - } - // activates/disables spinner for dynamic fields to indicate that they are currently being updated function wait(active) { for (var i in self.input_list) { diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 static/scripts/packed/mvc/tools/tools-form.js --- a/static/scripts/packed/mvc/tools/tools-form.js +++ b/static/scripts/packed/mvc/tools/tools-form.js @@ -1,1 +1,1 @@ -define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var d=b.extend({initialize:function(g){var f=this;this.job_handler=new a(this);this.buttons={execute:new e.Button({icon:"fa-check",tooltip:"Execute: "+g.name,title:"Execute",cls:"btn btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var f=this;var g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var i=top.location.href;var j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now you are using '"+f.options.name+"' version "+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught error.";f.modal.show({title:"Tool cannot be executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var h in g.inputs){var n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function m(r){for(var p in f.input_list){var q=f.field_list[p];var o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending current state (see below).");console.debug(g);var j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var u=f.field_list[p];if(u.update){var s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var r in t.options){var q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() - Received new model (see below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(i)}})}});return{View:d}}); \ No newline at end of file +define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var d=b.extend({initialize:function(g){var f=this;this.job_handler=new a(this);this.buttons={execute:new e.Button({icon:"fa-check",tooltip:"Execute: "+g.name,title:"Execute",cls:"btn btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var f=this;var g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var i=top.location.href;var j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now you are using '"+f.options.name+"' version "+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize() - Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught error.";f.modal.show({title:"Tool cannot be executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};function j(n){for(var l in f.input_list){var m=f.field_list[l];var k=f.input_list[l];if(k.is_dynamic&&m.wait&&m.unwait){if(n){m.wait()}else{m.unwait()}}}}j(true);var i=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending current state (see below).");console.debug(g);var h=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:h,data:g,success:function(k){f.tree.matchModel(k,function(m,q){var l=f.input_list[m];if(l&&l.options){if(!_.isEqual(l.options,q.options)){l.options=q.options;var r=f.field_list[m];if(r.update){var p=[];if((["data","data_collection","drill_down"]).indexOf(l.type)!=-1){p=l.options}else{for(var o in q.options){var n=q.options[o];if(n.length>2){p.push({label:n[0],value:n[1]})}}}r.update(p);r.trigger("change");console.debug("Updating options for "+m)}}}});j(false);console.debug("tools-form::_refreshForm() - Received new model (see below).");console.debug(k);f.deferred.done(i)},error:function(k){f.deferred.done(i);console.debug("tools-form::_refreshForm() - Refresh request failed.");console.debug(k)}})}});return{View:d}}); \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/46feb8f33ba9/ Changeset: 46feb8f33ba9 User: guerler Date: 2015-02-27 06:49:53+00:00 Summary: Parameters: Prevent uncaught exception on unavailable datasets in column validation Affected #: 1 file diff -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -1340,6 +1340,10 @@ datasets = util.listify( referent ) for dataset in datasets: if dataset: + # Check if metadata is available + if not hasattr(dataset, 'metadata'): + return True + # Check if the dataset does not have the expected metadata for columns if not dataset.metadata.columns: # Only allow late validation if the dataset is not yet ready https://bitbucket.org/galaxy/galaxy-central/commits/448272e06e42/ Changeset: 448272e06e42 User: guerler Date: 2015-02-27 06:57:39+00:00 Summary: ToolForm/Ui: Limit selection to available options Affected #: 3 files diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r 448272e06e42db47dec46a3a0dae58f9ca265160 client/galaxy/scripts/mvc/ui/ui-select-default.js --- a/client/galaxy/scripts/mvc/ui/ui-select-default.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js @@ -78,9 +78,11 @@ if (new_value === null) { new_value = '__null__'; } - this.$select.val(new_value); - if (this.$select.select2) { - this.$select.select2('val', new_value); + if (this.exists(new_value)) { + this.$select.val(new_value); + if (this.$select.select2) { + this.$select.select2('val', new_value); + } } } diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r 448272e06e42db47dec46a3a0dae58f9ca265160 static/scripts/mvc/ui/ui-select-default.js --- a/static/scripts/mvc/ui/ui-select-default.js +++ b/static/scripts/mvc/ui/ui-select-default.js @@ -78,9 +78,11 @@ if (new_value === null) { new_value = '__null__'; } - this.$select.val(new_value); - if (this.$select.select2) { - this.$select.select2('val', new_value); + if (this.exists(new_value)) { + this.$select.val(new_value); + if (this.$select.select2) { + this.$select.select2('val', new_value); + } } } diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r 448272e06e42db47dec46a3a0dae58f9ca265160 static/scripts/packed/mvc/ui/ui-select-default.js --- a/static/scripts/packed/mvc/ui/ui-select-default.js +++ b/static/scripts/packed/mvc/ui/ui-select-default.js @@ -1,1 +1,1 @@ -define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No data available",empty_text:"No selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}return this._getValue()},first:function(){var c=this.$select.find("option").first();if(c.length>0){return c.val()}else{return null}},text:function(){return this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-spinner fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-caret-down")},disabled:function(){return this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var d in c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div id="'+c.id+'" class="'+c.cls+'"><select id="'+c.id+'_select" class="select"/><div class="button"><i class="icon"/></div></div>'}});return{View:b}}); \ No newline at end of file +define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No data available",empty_text:"No selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return this._getValue()},first:function(){var c=this.$select.find("option").first();if(c.length>0){return c.val()}else{return null}},text:function(){return this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-spinner fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-caret-down")},disabled:function(){return this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var d in c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div id="'+c.id+'" class="'+c.cls+'"><select id="'+c.id+'_select" class="select"/><div class="button"><i class="icon"/></div></div>'}});return{View:b}}); \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/94298235c5c4/ Changeset: 94298235c5c4 User: guerler Date: 2015-02-27 06:59:43+00:00 Summary: Parameters: Fix deleted datasets validation message Affected #: 1 file diff -r 448272e06e42db47dec46a3a0dae58f9ca265160 -r 94298235c5c4df5f53bbfc5f7bb742837781001c lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2021,7 +2021,7 @@ for v in values: if v: if v.deleted: - raise ValueError( "The previously selected dataset has been previously deleted" ) + raise ValueError( "The previously selected dataset has been deleted." ) if hasattr( v, "dataset" ) and v.dataset.state in [ galaxy.model.Dataset.states.ERROR, galaxy.model.Dataset.states.DISCARDED ]: raise ValueError( "The previously selected dataset has entered an unusable state" ) if not self.multiple: https://bitbucket.org/galaxy/galaxy-central/commits/81c09bf641da/ Changeset: 81c09bf641da User: guerler Date: 2015-02-27 09:26:15+00:00 Summary: Parameters: Set hidden attribute for hidden dataset parameter Affected #: 1 file diff -r 94298235c5c4df5f53bbfc5f7bb742837781001c -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 lib/galaxy/tools/parameters/basic.py --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -2425,6 +2425,7 @@ DataToolParameter.__init__( self, tool, elem ) self.value = "None" self.type = "hidden_data" + self.hidden = True def get_initial_value( self, trans, context, history=None ): return None https://bitbucket.org/galaxy/galaxy-central/commits/b11c4470b151/ Changeset: b11c4470b151 User: guerler Date: 2015-02-27 09:36:20+00:00 Summary: ToolForm/Ui: Restrict available option requirement to single select fields Affected #: 3 files diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r b11c4470b151100c14f1a2abfb8047ac16b26f30 client/galaxy/scripts/mvc/ui/ui-select-default.js --- a/client/galaxy/scripts/mvc/ui/ui-select-default.js +++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js @@ -78,7 +78,7 @@ if (new_value === null) { new_value = '__null__'; } - if (this.exists(new_value)) { + if (this.exists(new_value) || this.options.multiple) { this.$select.val(new_value); if (this.$select.select2) { this.$select.select2('val', new_value); diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r b11c4470b151100c14f1a2abfb8047ac16b26f30 static/scripts/mvc/ui/ui-select-default.js --- a/static/scripts/mvc/ui/ui-select-default.js +++ b/static/scripts/mvc/ui/ui-select-default.js @@ -78,7 +78,7 @@ if (new_value === null) { new_value = '__null__'; } - if (this.exists(new_value)) { + if (this.exists(new_value) || this.options.multiple) { this.$select.val(new_value); if (this.$select.select2) { this.$select.select2('val', new_value); diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r b11c4470b151100c14f1a2abfb8047ac16b26f30 static/scripts/packed/mvc/ui/ui-select-default.js --- a/static/scripts/packed/mvc/ui/ui-select-default.js +++ b/static/scripts/packed/mvc/ui/ui-select-default.js @@ -1,1 +1,1 @@ -define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No data available",empty_text:"No selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return this._getValue()},first:function(){var c=this.$select.find("option").first();if(c.length>0){return c.val()}else{return null}},text:function(){return this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-spinner fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-caret-down")},disabled:function(){return this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var d in c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div id="'+c.id+'" class="'+c.cls+'"><select id="'+c.id+'_select" class="select"/><div class="button"><i class="icon"/></div></div>'}});return{View:b}}); \ No newline at end of file +define(["utils/utils"],function(a){var b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No data available",empty_text:"No selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)||this.options.multiple){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return this._getValue()},first:function(){var c=this.$select.find("option").first();if(c.length>0){return c.val()}else{return null}},text:function(){return this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-spinner fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa fa-caret-down")},disabled:function(){return this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var d in c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div id="'+c.id+'" class="'+c.cls+'"><select id="'+c.id+'_select" class="select"/><div class="button"><i class="icon"/></div></div>'}});return{View:b}}); \ No newline at end of file https://bitbucket.org/galaxy/galaxy-central/commits/6bd6c350296c/ Changeset: 6bd6c350296c User: carlfeberhard Date: 2015-02-27 15:03:47+00:00 Summary: Fix display_structure by using serializer; fix structure by removing extra decode_id Affected #: 2 files diff -r b11c4470b151100c14f1a2abfb8047ac16b26f30 -r 6bd6c350296c94817de06d3c239b9bf8d2b9978c lib/galaxy/webapps/galaxy/controllers/history.py --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -543,8 +543,10 @@ def structure( self, trans, id=None, **kwargs ): """ """ - id = self.decode_id( id ) if id else trans.history.id - history_to_view = self.history_manager.get_accessible( trans, id, trans.user ) + unencoded_history_id = trans.history.id + if id: + unencoded_history_id = self.decode_id( id ) + history_to_view = self.history_manager.get_accessible( trans, unencoded_history_id, trans.user ) history_data = self.history_manager._get_history_data( trans, history_to_view ) history_dictionary = history_data[ 'history' ] @@ -552,7 +554,7 @@ jobs = ( trans.sa_session.query( trans.app.model.Job ) .filter( trans.app.model.Job.user == trans.user ) - .filter( trans.app.model.Job.history_id == self.decode_id( id ) ) ).all() + .filter( trans.app.model.Job.history_id == unencoded_history_id ) ).all() jobs = map( lambda j: self.encode_all_ids( trans, j.to_dict( 'element' ), True ), jobs ) tools = {} diff -r b11c4470b151100c14f1a2abfb8047ac16b26f30 -r 6bd6c350296c94817de06d3c239b9bf8d2b9978c templates/webapps/galaxy/history/display_structured.mako --- a/templates/webapps/galaxy/history/display_structured.mako +++ b/templates/webapps/galaxy/history/display_structured.mako @@ -128,13 +128,15 @@ ## ---------------------------------------------------------------------------- <%def name="javascripts()"><% + from galaxy.managers import hdas + hda_serializer = hdas.HDASerializer( trans.app ) self.js_app = 'display-structured' controller = trans.webapp.controllers[ 'history' ] hda_dicts = [] id_hda_dict_map = {} for hda in history.active_datasets: - hda_dict = controller.get_hda_dict( trans, hda ) + hda_dict = hda_serializer.serialize_to_view( trans, hda, 'detailed' ) id_hda_dict_map[ hda_dict[ 'id' ] ] = hda_dict hda_dicts.append( hda_dict ) %> Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.