13 new commits in galaxy-central:
https://bitbucket.org/galaxy/galaxy-central/commits/0a6e7085ce60/
Changeset: 0a6e7085ce60
User: dan
Date: 2015-02-26 22:16:55+00:00
Summary: Allow TabularToolDataTable to be loaded from a url instead of a path.
Affected #: 1 file
diff -r 71596d6f10285ebb23d2a257eda3b70cb04ff655 -r
0a6e7085ce60c53e4a97aedd2e0561177b6f913a lib/galaxy/tools/data/__init__.py
--- a/lib/galaxy/tools/data/__init__.py
+++ b/lib/galaxy/tools/data/__init__.py
@@ -14,6 +14,8 @@
import hashlib
from glob import glob
+from tempfile import NamedTemporaryFile
+from urllib2 import urlopen
from galaxy import util
from galaxy.util.odict import odict
@@ -279,7 +281,22 @@
repo_info = None
# Read every file
for file_element in config_element.findall( 'file' ):
- filename = file_path = expand_here_template( file_element.get(
'path', None ), here=self.here )
+ tmp_file = None
+ filename = file_element.get( 'path', None )
+ if filename is None:
+ # Handle URLs as files
+ filename = file_element.get( 'url', None )
+ if filename:
+ tmp_file = NamedTemporaryFile( prefix='TTDT_URL_%s-' %
self.name )
+ try:
+ tmp_file.write( urlopen( filename ).read() )
+ except Exception, e:
+ log.error( 'Error loading Data Table URL "%s":
%s', filename, e )
+ continue
+ log.debug( 'Loading Data Table URL "%s" as filename
"%s".', filename, tmp_file.name )
+ filename = tmp_file.name
+ tmp_file.flush()
+ filename = file_path = expand_here_template( filename, here=self.here )
found = False
if file_path is None:
log.debug( "Encountered a file element (%s) that does not contain a
path value when loading tool data table '%s'.", util.xml_to_string(
file_element ), self.name )
@@ -325,6 +342,9 @@
config_element=config_element,
tool_shed_repository=repo_info, errors=errors )
else:
log.debug( "Filename '%s' already exists in filenames (%s),
not adding", filename, self.filenames.keys() )
+ # Remove URL tmp file
+ if tmp_file is not None:
+ tmp_file.close()
def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False,
persist_on_error=False, entry_source=None, **kwd ):
assert self.columns == other_table.columns, "Merging tabular data tables
with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name,
self.columns, other_table.name, other_table.columns )
https://bitbucket.org/galaxy/galaxy-central/commits/b6fe31e55f48/
Changeset: b6fe31e55f48
User: dannon
Date: 2015-02-27 00:59:26+00:00
Summary: Fix server_starttime being in app for reports, toolshed. I will be
refactoring this shortly to have a base UniverseApplication from which things inherit.
Affected #: 2 files
diff -r 0a6e7085ce60c53e4a97aedd2e0561177b6f913a -r
b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b lib/galaxy/webapps/reports/app.py
--- a/lib/galaxy/webapps/reports/app.py
+++ b/lib/galaxy/webapps/reports/app.py
@@ -1,5 +1,7 @@
+import config
import sys
-import config
+import time
+
import galaxy.model
from galaxy.web import security
@@ -26,6 +28,8 @@
self.targets_mysql = 'mysql' in self.config.database_connection
# Security helper
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
+ # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication
base.
+ self.server_starttime = int(time.time())
def shutdown( self ):
pass
diff -r 0a6e7085ce60c53e4a97aedd2e0561177b6f913a -r
b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b lib/galaxy/webapps/tool_shed/app.py
--- a/lib/galaxy/webapps/tool_shed/app.py
+++ b/lib/galaxy/webapps/tool_shed/app.py
@@ -1,17 +1,18 @@
import config
import sys
+import time
+import galaxy.datatypes.registry
+import galaxy.quota
+import galaxy.tools.data
+import galaxy.webapps.tool_shed.model
from galaxy import tools
-import galaxy.tools.data
-import galaxy.quota
-import galaxy.datatypes.registry
-import galaxy.webapps.tool_shed.model
+from galaxy.managers.tags import CommunityTagManager
from galaxy.openid.providers import OpenIDProviders
from galaxy.util.dbkeys import GenomeBuilds
from galaxy.web import security
-from galaxy.managers.tags import CommunityTagManager
-from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
import tool_shed.repository_registry
import tool_shed.repository_types.registry
+from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager
class UniverseApplication( object ):
@@ -70,6 +71,8 @@
self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir
# Initialize the repository registry.
self.repository_registry = tool_shed.repository_registry.Registry( self )
+ # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication
base.
+ self.server_starttime = int(time.time())
print >> sys.stderr, "Tool shed hgweb.config file is: ",
self.hgweb_config_manager.hgweb_config
def shutdown( self ):
https://bitbucket.org/galaxy/galaxy-central/commits/eed154f337fc/
Changeset: eed154f337fc
User: dannon
Date: 2015-02-27 01:13:48+00:00
Summary: Fix reports webapp under default database broken in ba825fc5
Affected #: 1 file
diff -r b6fe31e55f48916a4b83a5e9982dda3eab9a5a1b -r
eed154f337fc090168956bcbec92491d0a5d75c9 lib/galaxy/webapps/reports/app.py
--- a/lib/galaxy/webapps/reports/app.py
+++ b/lib/galaxy/webapps/reports/app.py
@@ -25,7 +25,10 @@
db_url,
self.config.database_engine_options,
create_tables=True )
- self.targets_mysql = 'mysql' in self.config.database_connection
+ if not self.config.database_connection:
+ self.targets_mysql = False
+ else:
+ self.targets_mysql = 'mysql' in self.config.database_connection
# Security helper
self.security = security.SecurityHelper( id_secret=self.config.id_secret )
# used for cachebusting -- refactor this into a *SINGLE* UniverseApplication
base.
https://bitbucket.org/galaxy/galaxy-central/commits/4e30957ab121/
Changeset: 4e30957ab121
User: jmchilton
Date: 2015-02-27 03:36:14+00:00
Summary: Fix incorrect usage of __all__ not containing strings...
...for plugin_config.py plugins.
Affected #: 12 files
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/core.py
--- a/lib/galaxy/jobs/metrics/instrumenters/core.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/core.py
@@ -82,4 +82,4 @@
pass
return value
-__all__ = [ CorePlugin ]
+__all__ = [ 'CorePlugin' ]
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py
--- a/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py
@@ -59,4 +59,4 @@
def __instrument_cpuinfo_path( self, job_directory ):
return self._instrument_file_path( job_directory, "cpuinfo" )
-__all__ = [ CpuInfoPlugin ]
+__all__ = [ 'CpuInfoPlugin' ]
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/env.py
--- a/lib/galaxy/jobs/metrics/instrumenters/env.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/env.py
@@ -68,4 +68,4 @@
def __env_file( self, job_directory ):
return self._instrument_file_path( job_directory, "vars" )
-__all__ = [ EnvPlugin ]
+__all__ = [ 'EnvPlugin' ]
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/meminfo.py
--- a/lib/galaxy/jobs/metrics/instrumenters/meminfo.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/meminfo.py
@@ -56,4 +56,4 @@
def __instrument_meminfo_path( self, job_directory ):
return self._instrument_file_path( job_directory, "meminfo" )
-__all__ = [ MemInfoPlugin ]
+__all__ = [ 'MemInfoPlugin' ]
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/jobs/metrics/instrumenters/uname.py
--- a/lib/galaxy/jobs/metrics/instrumenters/uname.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/uname.py
@@ -31,4 +31,4 @@
return self._instrument_file_path( job_directory, "uname" )
-__all__ = [ UnamePlugin ]
+__all__ = [ 'UnamePlugin' ]
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e
lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
--- a/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
+++ b/lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py
@@ -147,4 +147,4 @@
return base
-__all__ = [HomebrewToolShedDependencyResolver]
+__all__ = ['HomebrewToolShedDependencyResolver']
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e
lib/galaxy/tools/deps/resolvers/galaxy_packages.py
--- a/lib/galaxy/tools/deps/resolvers/galaxy_packages.py
+++ b/lib/galaxy/tools/deps/resolvers/galaxy_packages.py
@@ -71,4 +71,4 @@
commands = 'PACKAGE_BASE=%s; export PACKAGE_BASE; . %s' % (
base_path, self.script )
return commands
-__all__ = [GalaxyPackageDependencyResolver, GalaxyPackageDependency]
+__all__ = ['GalaxyPackageDependencyResolver', 'GalaxyPackageDependency']
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/homebrew.py
--- a/lib/galaxy/tools/deps/resolvers/homebrew.py
+++ b/lib/galaxy/tools/deps/resolvers/homebrew.py
@@ -53,4 +53,4 @@
return str( value ).lower() == "true"
-__all__ = [HomebrewDependencyResolver]
+__all__ = ['HomebrewDependencyResolver']
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/tools/deps/resolvers/modules.py
--- a/lib/galaxy/tools/deps/resolvers/modules.py
+++ b/lib/galaxy/tools/deps/resolvers/modules.py
@@ -153,4 +153,4 @@
def _string_as_bool( value ):
return str( value ).lower() == "true"
-__all__ = [ModuleDependencyResolver]
+__all__ = ['ModuleDependencyResolver']
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e
lib/galaxy/tools/deps/resolvers/tool_shed_packages.py
--- a/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py
+++ b/lib/galaxy/tools/deps/resolvers/tool_shed_packages.py
@@ -56,4 +56,4 @@
return INDETERMINATE_DEPENDENCY
-__all__ = [ToolShedPackageDependencyResolver]
+__all__ = ['ToolShedPackageDependencyResolver']
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/util/plugin_config.py
--- a/lib/galaxy/util/plugin_config.py
+++ b/lib/galaxy/util/plugin_config.py
@@ -25,6 +25,10 @@
# FIXME: this is not how one is suppose to use __all__ why did you do
# this past John?
for clazz in getattr( plugin_module, "__all__", [] ):
+ try:
+ clazz = getattr( plugin_module, clazz )
+ except TypeError:
+ clazz = clazz
plugin_type = getattr( clazz, plugin_type_identifier, None )
if plugin_type:
plugin_dict[ plugin_type ] = clazz
diff -r eed154f337fc090168956bcbec92491d0a5d75c9 -r
4e30957ab1217c50dbaed105d00b0a5afbea858e lib/galaxy/workflow/schedulers/core.py
--- a/lib/galaxy/workflow/schedulers/core.py
+++ b/lib/galaxy/workflow/schedulers/core.py
@@ -43,4 +43,4 @@
workflow_invocation=workflow_invocation,
)
-__all__ = [ CoreWorkflowSchedulingPlugin ]
+__all__ = [ 'CoreWorkflowSchedulingPlugin' ]
https://bitbucket.org/galaxy/galaxy-central/commits/e8c37fd94209/
Changeset: e8c37fd94209
User: jmchilton
Date: 2015-02-27 05:29:07+00:00
Summary: Update pulsar client and related modules.
Affected #: 24 files
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/cli.py
--- a/lib/galaxy/jobs/metrics/collectl/cli.py
+++ b/lib/galaxy/jobs/metrics/collectl/cli.py
@@ -12,7 +12,8 @@
class CollectlCli( object ):
- """ Abstraction over (some of) the command-line arguments of
collectl.
+ """
+ Abstraction over (some of) the command-line arguments of collectl.
Ideally this will be useful for building up command line arguments for
remote execution as well as runnning directly on local host.
@@ -20,33 +21,43 @@
collectl CLI - logic more directly related to the Galaxy job metric plugin
plugin should be placed in other modules.
- Keyword Arguments:
- collectl_path: Path to collectl executable (defaults to collectl - i.e.
- search the PATH).
+ **Keyword Arguments:**
- playback_path (defaults to None): If this is None collectl will run in
- record mode, else it will playback specified file.
+ ``collectl_path``
+ Path to collectl executable (defaults to collectl - i.e.
+ search the PATH).
- Playback Mode Options:
-
- sep : Separator used in playback mode (set to 9 to produce tsv)
- (defaults to None).
+ ``playback_path`` (defaults to ``None``)
+ If this is ``None``, collectl will run in
+ record mode, else it will playback specified file.
- Record Mode Options (some of these may work in playback mode also):
+ **Playback Mode Options:**
- destination_path: Location of path files to write to (defaults to None
- and collectl will just use cwd). Really this is just to prefix -
- collectl will append hostname and datetime to file.
- interval: Setup polling interval (secs) for most subsystems (defaults
- to None and when unspecified collectl will use default of 1 second).
- interval2: Setup polling interval (secs) for process information
- (defaults to None and when unspecified collectl will use default to
- 60 seconds).
- interval3: Setup polling interval (secs) for environment information
- (defaults to None and when unspecified collectl will use default to
- 300 seconds).
- procfilt: Optional argument to procfilt. (defaults to None).
- flush : Optional flush interval (defaults to None).
+ ``sep``
+ Separator used in playback mode (set to 9 to produce tsv)
+ (defaults to None).
+
+ **Record Mode Options** (some of these may work in playback mode also)
+
+ ``destination_path``
+ Location of path files to write to (defaults to None
+ and collectl will just use cwd). Really this is just to prefix -
+ collectl will append hostname and datetime to file.
+ ``interval``
+ Setup polling interval (secs) for most subsystems (defaults
+ to None and when unspecified collectl will use default of 1 second).
+ ``interval2``
+ Setup polling interval (secs) for process information
+ (defaults to None and when unspecified collectl will use default to
+ 60 seconds).
+ ``interval3``
+ Setup polling interval (secs) for environment information
+ (defaults to None and when unspecified collectl will use default to
+ 300 seconds).
+ ``procfilt``
+ Optional argument to procfilt. (defaults to None).
+ ``flush``
+ Optional flush interval (defaults to None).
"""
def __init__( self, **kwargs ):
@@ -125,4 +136,4 @@
if return_code:
raise Exception( "Problem running collectl command." )
-__all__ = [ CollectlCli ]
+__all__ = [ 'CollectlCli' ]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/processes.py
--- a/lib/galaxy/jobs/metrics/collectl/processes.py
+++ b/lib/galaxy/jobs/metrics/collectl/processes.py
@@ -26,7 +26,7 @@
#
# Process data dumped one row per process per interval.
-#http://collectl.sourceforge.net/Data-detail.html
+#
http://collectl.sourceforge.net/Data-detail.html
PROCESS_COLUMNS = [
"#Date", # Date of interval - e.g. 20140322
"Time", # Time of interval - 12:18:58
@@ -36,7 +36,7 @@
"PPID", # Parent PID of process.
"THRD", # Thread???
"S", # Process state - S - Sleeping, D - Uninterruptable Sleep, R -
Running, Z - Zombie or T - Stopped/Traced
- ## Memory options -
http://ewx.livejournal.com/579283.html
+ # Memory options -
http://ewx.livejournal.com/579283.html
"VmSize",
"VmLck",
"VmRSS",
@@ -93,7 +93,7 @@
statistics = DEFAULT_STATISTICS
statistics = util.listify( statistics )
- statistics = map( __tuplize_statistic, statistics )
+ statistics = map( _tuplize_statistic, statistics )
# Check for validity...
for statistic in statistics:
if statistic[ 0 ] not in STATISTIC_TYPES:
@@ -109,10 +109,10 @@
with tempfile.NamedTemporaryFile( ) as tmp_tsv:
collectl_playback_cli.run( stdout=tmp_tsv )
with open( tmp_tsv.name, "r" ) as tsv_file:
- return __read_process_statistics( tsv_file, pid, statistics )
+ return _read_process_statistics( tsv_file, pid, statistics )
-def __read_process_statistics( tsv_file, pid, statistics ):
+def _read_process_statistics( tsv_file, pid, statistics ):
process_summarizer = CollectlProcessSummarizer( pid, statistics )
current_interval = None
@@ -167,7 +167,7 @@
to_num = float
else:
to_num = long
-
+
interval_stat = sum( to_num( r[ column_index ] ) for r in rows )
self.tree_statistics[ column_name ].track( interval_stat )
@@ -228,7 +228,7 @@
ability to filter out just rows corresponding to the process tree
corresponding to a given pid.
"""
-
+
def __init__( self ):
self.rows = []
@@ -242,11 +242,11 @@
self.rows.append( row )
-def __tuplize_statistic( statistic ):
+def _tuplize_statistic( statistic ):
if not isinstance( statistic, tuple ):
statistic_split = statistic.split( "_", 1 )
statistic = ( statistic_split[ 0 ].lower(), statistic_split[ 1 ] )
return statistic
-__all__ = [ generate_process_statistics ]
+__all__ = [ 'generate_process_statistics' ]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/metrics/collectl/subsystems.py
--- a/lib/galaxy/jobs/metrics/collectl/subsystems.py
+++ b/lib/galaxy/jobs/metrics/collectl/subsystems.py
@@ -69,4 +69,4 @@
"""
return SUBSYSTEM_DICT[ name ]
-__all__ = [ get_subsystem ]
+__all__ = [ 'get_subsystem' ]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce
lib/galaxy/jobs/metrics/instrumenters/collectl.py
--- a/lib/galaxy/jobs/metrics/instrumenters/collectl.py
+++ b/lib/galaxy/jobs/metrics/instrumenters/collectl.py
@@ -211,4 +211,4 @@
return ""
-__all__ = [ CollectlPlugin ]
+__all__ = [ 'CollectlPlugin' ]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/__init__.py
--- a/lib/galaxy/jobs/runners/util/__init__.py
+++ b/lib/galaxy/jobs/runners/util/__init__.py
@@ -7,4 +7,4 @@
from .kill import kill_pid
-__all__ = [kill_pid, Bunch]
+__all__ = ['kill_pid', 'Bunch']
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/__init__.py
--- a/lib/galaxy/jobs/runners/util/cli/__init__.py
+++ b/lib/galaxy/jobs/runners/util/cli/__init__.py
@@ -6,6 +6,9 @@
DEFAULT_SHELL_PLUGIN = 'LocalShell'
+ERROR_MESSAGE_NO_JOB_PLUGIN = "No job plugin parameter found, cannot create CLI job
interface"
+ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN = "Failed to find job_plugin of type %s, available
types include %s"
+
class CliInterface(object):
"""
@@ -53,8 +56,14 @@
return shell
def get_job_interface(self, job_params):
- job_plugin = job_params['plugin']
- job_interface = self.cli_job_interfaces[job_plugin](**job_params)
+ job_plugin = job_params.get('plugin', None)
+ if not job_plugin:
+ raise ValueError(ERROR_MESSAGE_NO_JOB_PLUGIN)
+ job_plugin_class = self.cli_job_interfaces.get(job_plugin, None)
+ if not job_plugin_class:
+ raise ValueError(ERROR_MESSAGE_NO_SUCH_JOB_PLUGIN % (job_plugin,
self.cli_job_interfaces.keys()))
+ job_interface = job_plugin_class(**job_params)
+
return job_interface
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/factory.py
--- a/lib/galaxy/jobs/runners/util/cli/factory.py
+++ b/lib/galaxy/jobs/runners/util/cli/factory.py
@@ -12,7 +12,11 @@
code_dir = '.'
+def build_cli_interface():
+ return CliInterface(code_dir=code_dir)
+
+
def get_shell(params):
- cli_interface = CliInterface(code_dir=code_dir)
+ cli_interface = build_cli_interface()
shell_params, _ = split_params(params)
return cli_interface.get_shell_plugin(shell_params)
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/job/slurm.py
--- a/lib/galaxy/jobs/runners/util/cli/job/slurm.py
+++ b/lib/galaxy/jobs/runners/util/cli/job/slurm.py
@@ -1,6 +1,10 @@
# A simple CLI runner for slurm that can be used when running Galaxy from a
# non-submit host and using a Slurm cluster.
+from ..job import BaseJobExec
+
+from logging import getLogger
+
try:
from galaxy.model import Job
job_states = Job.states
@@ -9,11 +13,6 @@
from galaxy.util import enum
job_states = enum(RUNNING='running', OK='complete',
QUEUED='queued', ERROR="failed")
-from ..job import BaseJobExec
-
-__all__ = ('Slurm',)
-
-from logging import getLogger
log = getLogger(__name__)
argmap = {
@@ -94,3 +93,6 @@
}.get(state)
except KeyError:
raise KeyError("Failed to map slurm status code [%s] to job state."
% state)
+
+
+__all__ = ('Slurm',)
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/job/torque.py
--- a/lib/galaxy/jobs/runners/util/cli/job/torque.py
+++ b/lib/galaxy/jobs/runners/util/cli/job/torque.py
@@ -1,3 +1,4 @@
+from logging import getLogger
try:
import xml.etree.cElementTree as et
except:
@@ -13,10 +14,10 @@
from ..job import BaseJobExec
-__all__ = ('Torque',)
+log = getLogger(__name__)
-from logging import getLogger
-log = getLogger(__name__)
+ERROR_MESSAGE_UNRECOGNIZED_ARG = 'Unrecognized long argument passed to Torque CLI
plugin: %s'
+
argmap = {'destination': '-q',
'Execution_Time': '-a',
@@ -58,8 +59,8 @@
if not k.startswith('-'):
k = argmap[k]
pbsargs[k] = v
- except:
- log.warning('Unrecognized long argument passed to Torque CLI plugin:
%s' % k)
+ except KeyError:
+ log.warning(ERROR_MESSAGE_UNRECOGNIZED_ARG % k)
template_pbsargs = ''
for k, v in pbsargs.items():
template_pbsargs += '#PBS %s %s\n' % (k, v)
@@ -118,3 +119,6 @@
}.get(state)
except KeyError:
raise KeyError("Failed to map torque status code [%s] to job
state." % state)
+
+
+__all__ = ('Torque',)
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/galaxy/jobs/runners/util/cli/shell/local.py
--- a/lib/galaxy/jobs/runners/util/cli/shell/local.py
+++ b/lib/galaxy/jobs/runners/util/cli/shell/local.py
@@ -24,7 +24,7 @@
True
>> exec_result.stdout.strip() == u'Hello World'
True
- >>> exec_result = exec_python("import time; time.sleep(90)",
timeout=3, timeout_check_interval=1)
+ >>> exec_result = exec_python("import time; time.sleep(90)",
timeout=1, timeout_check_interval=.1)
>> exec_result.stdout == u''
True
>> exec_result.stderr == 'Execution timed out'
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/__init__.py
--- a/lib/pulsar/client/__init__.py
+++ b/lib/pulsar/client/__init__.py
@@ -1,6 +1,6 @@
"""
pulsar client
-======
+=================
This module contains logic for interfacing with an external Pulsar server.
@@ -50,13 +50,13 @@
from .path_mapper import PathMapper
__all__ = [
- build_client_manager,
- OutputNotFoundException,
- url_to_destination_params,
- finish_job,
- submit_job,
- ClientJobDescription,
- PulsarOutputs,
- ClientOutputs,
- PathMapper,
+ 'build_client_manager',
+ 'OutputNotFoundException',
+ 'url_to_destination_params',
+ 'finish_job',
+ 'submit_job',
+ 'ClientJobDescription',
+ 'PulsarOutputs',
+ 'ClientOutputs',
+ 'PathMapper',
]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/action_mapper.py
--- a/lib/pulsar/client/action_mapper.py
+++ b/lib/pulsar/client/action_mapper.py
@@ -1,5 +1,6 @@
from json import load
from os import makedirs
+from os import unlink
from os.path import exists
from os.path import abspath
from os.path import dirname
@@ -16,6 +17,9 @@
from .util import unique_path_prefix
from .transport import get_file
from .transport import post_file
+from .transport import rsync_get_file, scp_get_file
+from .transport import rsync_post_file, scp_post_file
+import tempfile
DEFAULT_MAPPED_ACTION = 'transfer' # Not really clear to me what this should be,
exception?
@@ -130,6 +134,7 @@
if config is None:
config = self.__client_to_config(client)
self.default_action = config.get("default_action",
"transfer")
+ self.ssh_key = config.get("ssh_key", None)
self.mappers = mappers_from_dicts(config.get("paths", []))
self.files_endpoint = config.get("files_endpoint", None)
@@ -155,6 +160,7 @@
return dict(
default_action=self.default_action,
files_endpoint=self.files_endpoint,
+ ssh_key=self.ssh_key,
paths=map(lambda m: m.to_dict(), self.mappers)
)
@@ -166,6 +172,8 @@
config = dict()
config["default_action"] = client.default_file_action
config["files_endpoint"] = client.files_endpoint
+ if hasattr(client, 'ssh_key'):
+ config["ssh_key"] = client.ssh_key
return config
def __load_action_config(self, path):
@@ -209,6 +217,9 @@
# TODO: URL encode path.
url = "%s&path=%s&file_type=%s" % (url_base, action.path,
file_type)
action.url = url
+ elif action.action_type in ["remote_rsync_transfer",
"remote_scp_transfer"]:
+ # Required, so no check for presence
+ action.ssh_key = self.ssh_key
REQUIRED_ACTION_KWD = object()
@@ -375,6 +386,95 @@
post_file(self.url, pulsar_path)
+class PubkeyAuthenticatedTransferAction(BaseAction):
+ """Base class for file transfers requiring an SSH public/private key
+ """
+ action_spec = dict(
+ ssh_user=REQUIRED_ACTION_KWD,
+ ssh_host=REQUIRED_ACTION_KWD,
+ ssh_port=REQUIRED_ACTION_KWD,
+ )
+ action_type = "remote_pubkey_transfer"
+ staging = STAGING_ACTION_REMOTE
+ ssh_key = None
+
+ def __init__(self, path, file_lister=None, url=None, ssh_user=None,
+ ssh_host=None, ssh_port=None, ssh_key=None):
+ super(PubkeyAuthenticatedTransferAction, self).__init__(path,
file_lister=file_lister)
+ self.url = url
+ self.ssh_user = ssh_user
+ self.ssh_host = ssh_host
+ self.ssh_port = ssh_port
+ self.ssh_key = ssh_key
+
+ def to_dict(self):
+ return dict(path=self.path, action_type=self.action_type, url=self.url,
+ ssh_user=self.ssh_user, ssh_host=self.ssh_host,
+ ssh_port=self.ssh_port)
+
+ def serialize_ssh_key(self):
+ f = tempfile.NamedTemporaryFile(delete=False)
+ if self.ssh_key is not None:
+ f.write(self.ssh_key)
+ else:
+ raise Exception("SSH_KEY not available")
+ return f.name
+
+ def cleanup_ssh_key(self, keyfile):
+ if exists(keyfile):
+ unlink(keyfile)
+
+
+class RsyncTransferAction(PubkeyAuthenticatedTransferAction):
+ action_type = "remote_rsync_transfer"
+
+ @classmethod
+ def from_dict(cls, action_dict):
+ return RsyncTransferAction(path=action_dict["path"],
+ url=action_dict["url"],
+ ssh_user=action_dict["ssh_user"],
+ ssh_host=action_dict["ssh_host"],
+ ssh_port=action_dict["ssh_port"],
+ ssh_key=action_dict["ssh_key"])
+
+ def write_to_path(self, path):
+ key_file = self.serialize_ssh_key()
+ rsync_get_file(self.path, path, self.ssh_user, self.ssh_host,
+ self.ssh_port, key_file)
+ self.cleanup_ssh_key(key_file)
+
+ def write_from_path(self, pulsar_path):
+ key_file = self.serialize_ssh_key()
+ rsync_post_file(pulsar_path, self.path, self.ssh_user,
+ self.ssh_host, self.ssh_port, key_file)
+ self.cleanup_ssh_key(key_file)
+
+
+class ScpTransferAction(PubkeyAuthenticatedTransferAction):
+ action_type = "remote_scp_transfer"
+
+ @classmethod
+ def from_dict(cls, action_dict):
+ return ScpTransferAction(path=action_dict["path"],
+ url=action_dict["url"],
+ ssh_user=action_dict["ssh_user"],
+ ssh_host=action_dict["ssh_host"],
+ ssh_port=action_dict["ssh_port"],
+ ssh_key=action_dict["ssh_key"])
+
+ def write_to_path(self, path):
+ key_file = self.serialize_ssh_key()
+ scp_get_file(self.path, path, self.ssh_user, self.ssh_host,
+ self.ssh_port, key_file)
+ self.cleanup_ssh_key(key_file)
+
+ def write_from_path(self, pulsar_path):
+ key_file = self.serialize_ssh_key()
+ scp_post_file(pulsar_path, self.path, self.ssh_user, self.ssh_host,
+ self.ssh_port, key_file)
+ self.cleanup_ssh_key(key_file)
+
+
class MessageAction(object):
""" Sort of pseudo action describing "files" store in memory
and
transferred via message (HTTP, Python-call, MQ, etc...)
@@ -408,7 +508,7 @@
open(path, "w").write(self.contents)
-DICTIFIABLE_ACTION_CLASSES = [RemoteCopyAction, RemoteTransferAction, MessageAction]
+DICTIFIABLE_ACTION_CLASSES = [RemoteCopyAction, RemoteTransferAction, MessageAction,
RsyncTransferAction, ScpTransferAction]
def from_dict(action_dict):
@@ -517,10 +617,10 @@
def mappers_from_dicts(mapper_def_list):
- return map(lambda m: __mappper_from_dict(m), mapper_def_list)
+ return map(lambda m: _mappper_from_dict(m), mapper_def_list)
-def __mappper_from_dict(mapper_dict):
+def _mappper_from_dict(mapper_dict):
map_type = mapper_dict.get('match_type', DEFAULT_PATH_MAPPER_TYPE)
return MAPPER_CLASS_DICT[map_type](mapper_dict)
@@ -554,14 +654,16 @@
CopyAction,
RemoteCopyAction,
RemoteTransferAction,
+ RsyncTransferAction,
+ ScpTransferAction,
]
actions = dict([(clazz.action_type, clazz) for clazz in ACTION_CLASSES])
__all__ = [
- FileActionMapper,
- path_type,
- from_dict,
- MessageAction,
- RemoteTransferAction, # For testing
+ 'FileActionMapper',
+ 'path_type',
+ 'from_dict',
+ 'MessageAction',
+ 'RemoteTransferAction', # For testing
]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/amqp_exchange.py
--- a/lib/pulsar/client/amqp_exchange.py
+++ b/lib/pulsar/client/amqp_exchange.py
@@ -1,13 +1,16 @@
+import copy
+import uuid
+import socket
+import logging
+import threading
+from time import sleep
+
try:
import kombu
from kombu import pools
except ImportError:
kombu = None
-import socket
-import logging
-import threading
-from time import sleep
log = logging.getLogger(__name__)
@@ -101,24 +104,45 @@
log.debug('AMQP heartbeat thread exiting')
def publish(self, name, payload):
+ # Consider optionally disabling if throughput becomes main concern.
+ transaction_uuid = uuid.uuid1()
key = self.__queue_name(name)
- log.debug("Begin publishing to key %s" % key)
+ publish_log_prefix = self.__publish_log_prefex(transaction_uuid)
+ log.debug("%sBegin publishing to key %s", publish_log_prefix, key)
with self.connection(self.__url) as connection:
with pools.producers[connection].acquire() as producer:
- log.debug("Have producer for publishing to key %s" % key)
+ log.debug("%sHave producer for publishing to key %s",
publish_log_prefix, key)
+ publish_kwds = self.__prepare_publish_kwds(publish_log_prefix)
producer.publish(
payload,
serializer='json',
exchange=self.__exchange,
declare=[self.__exchange],
routing_key=key,
- **self.__publish_kwds
+ **publish_kwds
)
- log.debug("Published to key %s" % key)
+ log.debug("%sPublished to key %s", publish_log_prefix, key)
- def __publish_errback(self, exc, interval):
- log.error("Connection error while publishing: %r", exc, exc_info=1)
- log.info("Retrying in %s seconds", interval)
+ def __prepare_publish_kwds(self, publish_log_prefix):
+ if "retry_policy" in self.__publish_kwds:
+ publish_kwds = copy.deepcopy(self.__publish_kwds)
+
+ def errback(exc, interval):
+ return self.__publish_errback(exc, interval, publish_log_prefix)
+ publish_kwds["retry_policy"]["errback"] = errback
+ else:
+ publish_kwds = self.__publish_kwds
+ return publish_kwds
+
+ def __publish_errback(self, exc, interval, publish_log_prefix=""):
+ log.error("%sConnection error while publishing: %r",
publish_log_prefix, exc, exc_info=1)
+ log.info("%sRetrying in %s seconds", publish_log_prefix, interval)
+
+ def __publish_log_prefex(self, transaction_uuid=None):
+ prefix = ""
+ if transaction_uuid:
+ prefix = "[publish:%s] " % str(transaction_uuid)
+ return prefix
def connection(self, connection_string, **kwargs):
if "ssl" not in kwargs:
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/client.py
--- a/lib/pulsar/client/client.py
+++ b/lib/pulsar/client/client.py
@@ -41,6 +41,12 @@
)
else:
job_directory = None
+
+ if "ssh_key" in (destination_params or {}):
+ self.ssh_key = destination_params["ssh_key"]
+ else:
+ self.ssh_key = None
+
self.env = destination_params.get("env", [])
self.files_endpoint = destination_params.get("files_endpoint", None)
self.job_directory = job_directory
@@ -271,6 +277,7 @@
launch_params['env'] = env
if remote_staging:
launch_params['remote_staging'] = remote_staging
+ launch_params['remote_staging']['ssh_key'] = self.ssh_key
if job_config and self.setup_handler.local:
# Setup not yet called, job properties were inferred from
# destination arguments. Hence, must have Pulsar setup job
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/manager.py
--- a/lib/pulsar/client/manager.py
+++ b/lib/pulsar/client/manager.py
@@ -220,4 +220,8 @@
int_val = int(val)
return int_val
-__all__ = [ClientManager, ObjectStoreClientManager, HttpPulsarInterface]
+__all__ = [
+ 'ClientManager',
+ 'ObjectStoreClientManager',
+ 'HttpPulsarInterface'
+]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/path_mapper.py
--- a/lib/pulsar/client/path_mapper.py
+++ b/lib/pulsar/client/path_mapper.py
@@ -95,4 +95,4 @@
message = "PathMapper cannot handle path type %s" %
dataset_path_type
raise Exception(message)
-__all__ = [PathMapper]
+__all__ = ['PathMapper']
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/setup_handler.py
--- a/lib/pulsar/client/setup_handler.py
+++ b/lib/pulsar/client/setup_handler.py
@@ -82,6 +82,7 @@
unstructured_files_directory = job_directory.unstructured_files_directory()
sep = system_properties.get("sep", os.sep)
job_config = {
+ "job_directory": job_directory.path,
"working_directory": working_directory,
"outputs_directory": outputs_directory,
"configs_directory": configs_directory,
@@ -100,4 +101,4 @@
return job_config
-__all__ = [build_job_config, build]
+__all__ = ['build_job_config', 'build']
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/staging/down.py
--- a/lib/pulsar/client/staging/down.py
+++ b/lib/pulsar/client/staging/down.py
@@ -152,4 +152,4 @@
except Exception:
log.warn("Failed to cleanup remote Pulsar job")
-__all__ = [finish_job]
+__all__ = ['finish_job']
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/staging/up.py
--- a/lib/pulsar/client/staging/up.py
+++ b/lib/pulsar/client/staging/up.py
@@ -349,14 +349,18 @@
local_action = action.staging_action_local
if local_action:
response = self.client.put_file(path, type, name=name,
contents=contents)
- get_path = lambda: response['path']
+
+ def get_path():
+ return response['path']
else:
job_directory = self.client.job_directory
assert job_directory, "job directory required for action %s" %
action
if not name:
name = basename(path)
self.__add_remote_staging_input(action, name, type)
- get_path = lambda: job_directory.calculate_path(name, type)
+
+ def get_path():
+ return job_directory.calculate_path(name, type)
register = self.rewrite_paths or type == 'tool' # Even if inputs not
rewritten, tool must be.
if register:
self.register_rewrite(path, get_path(), type, force=True)
@@ -417,4 +421,4 @@
input.close()
-__all__ = [submit_job]
+__all__ = ['submit_job']
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/__init__.py
--- a/lib/pulsar/client/transport/__init__.py
+++ b/lib/pulsar/client/transport/__init__.py
@@ -2,26 +2,8 @@
from .curl import PycurlTransport
import os
-
-def get_transport(transport_type=None, os_module=os):
- transport_type = __get_transport_type(transport_type, os_module)
- if transport_type == 'urllib':
- transport = Urllib2Transport()
- else:
- transport = PycurlTransport()
- return transport
-
-
-def __get_transport_type(transport_type, os_module):
- if not transport_type:
- use_curl = os_module.getenv('PULSAR_CURL_TRANSPORT', "0")
- # If PULSAR_CURL_TRANSPORT is unset or set to 0, use default,
- # else use curl.
- if use_curl.isdigit() and not int(use_curl):
- transport_type = 'urllib'
- else:
- transport_type = 'curl'
- return transport_type
+from .ssh import rsync_get_file, scp_get_file
+from .ssh import rsync_post_file, scp_post_file
from .curl import curl_available
from .requests import requests_multipart_post_available
@@ -35,4 +17,33 @@
from .poster import get_file
from .poster import post_file
-__all__ = [get_transport, get_file, post_file]
+
+def get_transport(transport_type=None, os_module=os):
+ transport_type = _get_transport_type(transport_type, os_module)
+ if transport_type == 'urllib':
+ transport = Urllib2Transport()
+ else:
+ transport = PycurlTransport()
+ return transport
+
+
+def _get_transport_type(transport_type, os_module):
+ if not transport_type:
+ use_curl = os_module.getenv('PULSAR_CURL_TRANSPORT', "0")
+ # If PULSAR_CURL_TRANSPORT is unset or set to 0, use default,
+ # else use curl.
+ if use_curl.isdigit() and not int(use_curl):
+ transport_type = 'urllib'
+ else:
+ transport_type = 'curl'
+ return transport_type
+
+__all__ = [
+ 'get_transport',
+ 'get_file',
+ 'post_file',
+ 'rsync_get_file',
+ 'rsync_post_file',
+ 'scp_get_file',
+ 'scp_post_file'
+]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/curl.py
--- a/lib/pulsar/client/transport/curl.py
+++ b/lib/pulsar/client/transport/curl.py
@@ -2,32 +2,37 @@
from cStringIO import StringIO
except ImportError:
from io import StringIO
-curl_available = True
+
try:
- from pycurl import Curl
+ from pycurl import Curl, HTTP_CODE
+ curl_available = True
except ImportError:
curl_available = False
-from os.path import getsize
+
+import os.path
PYCURL_UNAVAILABLE_MESSAGE = \
"You are attempting to use the Pycurl version of the Pulsar client but pycurl is
unavailable."
+NO_SUCH_FILE_MESSAGE = "Attempt to post file %s to URL %s, but file does not
exist."
+POST_FAILED_MESSAGE = "Failed to post_file properly for url %s, remote server
returned status code of %s."
+GET_FAILED_MESSAGE = "Failed to get_file properly for url %s, remote server returned
status code of %s."
+
class PycurlTransport(object):
def execute(self, url, method=None, data=None, input_path=None, output_path=None):
buf = _open_output(output_path)
try:
- c = _new_curl_object()
- c.setopt(c.URL, url.encode('ascii'))
+ c = _new_curl_object_for_url(url)
c.setopt(c.WRITEFUNCTION, buf.write)
if method:
c.setopt(c.CUSTOMREQUEST, method)
if input_path:
c.setopt(c.UPLOAD, 1)
c.setopt(c.READFUNCTION, open(input_path, 'rb').read)
- filesize = getsize(input_path)
+ filesize = os.path.getsize(input_path)
c.setopt(c.INFILESIZE, filesize)
if data:
c.setopt(c.POST, 1)
@@ -42,19 +47,30 @@
def post_file(url, path):
- c = _new_curl_object()
- c.setopt(c.URL, url.encode('ascii'))
+ if not os.path.exists(path):
+ # pycurl doesn't always produce a great exception for this,
+ # wrap it in a better one.
+ message = NO_SUCH_FILE_MESSAGE % (path, url)
+ raise Exception(message)
+ c = _new_curl_object_for_url(url)
c.setopt(c.HTTPPOST, [("file", (c.FORM_FILE,
path.encode('ascii')))])
c.perform()
+ status_code = c.getinfo(HTTP_CODE)
+ if int(status_code) != 200:
+ message = POST_FAILED_MESSAGE % (url, status_code)
+ raise Exception(message)
def get_file(url, path):
buf = _open_output(path)
try:
- c = _new_curl_object()
- c.setopt(c.URL, url.encode('ascii'))
+ c = _new_curl_object_for_url(url)
c.setopt(c.WRITEFUNCTION, buf.write)
c.perform()
+ status_code = c.getinfo(HTTP_CODE)
+ if int(status_code) != 200:
+ message = GET_FAILED_MESSAGE % (url, status_code)
+ raise Exception(message)
finally:
buf.close()
@@ -63,10 +79,20 @@
return open(output_path, 'wb') if output_path else StringIO()
+def _new_curl_object_for_url(url):
+ c = _new_curl_object()
+ c.setopt(c.URL, url.encode('ascii'))
+ return c
+
+
def _new_curl_object():
try:
return Curl()
except NameError:
raise ImportError(PYCURL_UNAVAILABLE_MESSAGE)
-___all__ = [PycurlTransport, post_file, get_file]
+__all__ = [
+ 'PycurlTransport',
+ 'post_file',
+ 'get_file'
+]
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/poster.py
--- a/lib/pulsar/client/transport/poster.py
+++ b/lib/pulsar/client/transport/poster.py
@@ -1,4 +1,7 @@
from __future__ import absolute_import
+
+import logging
+
try:
from urllib2 import urlopen
except ImportError:
@@ -20,7 +23,6 @@
POSTER_UNAVAILABLE_MESSAGE = "Pulsar configured to use poster module - but it is
unavailable. Please install poster."
-import logging
log = logging.getLogger(__name__)
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/requests.py
--- a/lib/pulsar/client/transport/requests.py
+++ b/lib/pulsar/client/transport/requests.py
@@ -1,4 +1,7 @@
from __future__ import absolute_import
+
+import logging
+
try:
from galaxy import eggs
eggs.require("requests")
@@ -9,18 +12,18 @@
import requests
except ImportError:
requests = None
-requests_multipart_post_available = False
+
try:
import requests_toolbelt
requests_multipart_post_available = True
except ImportError:
+ requests_multipart_post_available = False
requests_toolbelt = None
REQUESTS_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests module - but it is
unavailable. Please install requests."
REQUESTS_TOOLBELT_UNAVAILABLE_MESSAGE = "Pulsar configured to use requests_toolbelt
module - but it is unavailable. Please install requests_toolbelt."
-import logging
log = logging.getLogger(__name__)
diff -r 4e30957ab1217c50dbaed105d00b0a5afbea858e -r
e8c37fd942090370354baee82a6baf9fdc0ac0ce lib/pulsar/client/transport/ssh.py
--- /dev/null
+++ b/lib/pulsar/client/transport/ssh.py
@@ -0,0 +1,64 @@
+import subprocess
+SSH_OPTIONS = ('-o', 'StrictHostKeyChecking=no', '-o',
'PreferredAuthentications=publickey', '-o',
'PubkeyAuthentication=yes')
+
+
+def rsync_get_file(uri_from, uri_to, user, host, port, key):
+ cmd = [
+ 'rsync',
+ '-e',
+ 'ssh -i %s -p %s %s' % (key, port, ' '.join(SSH_OPTIONS)),
+ '%s@%s:%s' % (user, host, uri_from),
+ uri_to,
+ ]
+ exit_code = subprocess.check_call(cmd)
+ if exit_code != 0:
+ raise Exception("Rsync exited with code %s" % exit_code)
+
+
+def rsync_post_file(uri_from, uri_to, user, host, port, key):
+ cmd = [
+ 'rsync',
+ '-e',
+ 'ssh -i %s -p %s %s' % (key, port, ' '.join(SSH_OPTIONS)),
+ uri_from,
+ '%s@%s:%s' % (user, host, uri_to),
+ ]
+ exit_code = subprocess.check_call(cmd)
+ if exit_code != 0:
+ raise Exception("Rsync exited with code %s" % exit_code)
+
+
+def scp_get_file(uri_from, uri_to, user, host, port, key):
+ cmd = [
+ 'scp',
+ '-P', str(port),
+ '-i', key,
+ SSH_OPTIONS,
+ '%s@%s:%s' % (user, host, uri_from),
+ uri_to,
+ ]
+ exit_code = subprocess.check_call(cmd)
+ if exit_code != 0:
+ raise Exception("scp exited with code %s" % exit_code)
+
+
+def scp_post_file(uri_from, uri_to, user, host, port, key):
+ cmd = [
+ 'scp',
+ '-P', str(port),
+ '-i', key,
+ SSH_OPTIONS,
+ uri_from,
+ '%s@%s:%s' % (user, host, uri_to),
+ ]
+ exit_code = subprocess.check_call(cmd)
+ if exit_code != 0:
+ raise Exception("scp exited with code %s" % exit_code)
+
+
+___all__ = [
+ 'rsync_post_file',
+ 'rsync_get_file',
+ 'scp_post_file',
+ 'scp_get_file'
+]
https://bitbucket.org/galaxy/galaxy-central/commits/e630e74d5571/
Changeset: e630e74d5571
User: guerler
Date: 2015-02-27 06:30:48+00:00
Summary: ToolForm: Show initial validation errors on re-run
Affected #: 3 files
diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r
e630e74d5571462111045b7db9b021427c7ab6ef client/galaxy/scripts/mvc/tools/tools-form.js
--- a/client/galaxy/scripts/mvc/tools/tools-form.js
+++ b/client/galaxy/scripts/mvc/tools/tools-form.js
@@ -22,6 +22,7 @@
}
})
}
+ options.initial_errors = options.job_id;
ToolFormBase.prototype.initialize.call(this, options);
},
diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r
e630e74d5571462111045b7db9b021427c7ab6ef static/scripts/mvc/tools/tools-form.js
--- a/static/scripts/mvc/tools/tools-form.js
+++ b/static/scripts/mvc/tools/tools-form.js
@@ -22,6 +22,7 @@
}
})
}
+ options.initial_errors = options.job_id;
ToolFormBase.prototype.initialize.call(this, options);
},
diff -r e8c37fd942090370354baee82a6baf9fdc0ac0ce -r
e630e74d5571462111045b7db9b021427c7ab6ef static/scripts/packed/mvc/tools/tools-form.js
--- a/static/scripts/packed/mvc/tools/tools-form.js
+++ b/static/scripts/packed/mvc/tools/tools-form.js
@@ -1,1 +1,1 @@
-define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var
d=b.extend({initialize:function(g){var f=this;this.job_handler=new
a(this);this.buttons={execute:new
e.Button({icon:"fa-check",tooltip:"Execute:
"+g.name,title:"Execute",cls:"btn
btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};b.prototype.initialize.call(this,g)},_buildModel:function(){var
f=this;var
g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var
i=top.location.href;var
j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var
h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now
you are using '"+f.options.name+"' version
"+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model
ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught
error.";f.modal.show({title:"Tool cannot be
executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var
f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var
h in g.inputs){var
n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function
m(r){for(var p in f.input_list){var q=f.field_list[p];var
o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var
l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending
current state (see below).");console.debug(g);var
j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var
o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var
u=f.field_list[p];if(u.update){var
s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var
r in t.options){var
q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating
options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() -
Received new model (see
below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm()
- Refresh request failed.");console.debug(i)}})}});return{View:d}});
\ No newline at end of file
+define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var
d=b.extend({initialize:function(g){var f=this;this.job_handler=new
a(this);this.buttons={execute:new
e.Button({icon:"fa-check",tooltip:"Execute:
"+g.name,title:"Execute",cls:"btn
btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var
f=this;var
g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var
i=top.location.href;var
j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var
h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now
you are using '"+f.options.name+"' version
"+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model
ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught
error.";f.modal.show({title:"Tool cannot be
executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var
f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var
h in g.inputs){var
n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function
m(r){for(var p in f.input_list){var q=f.field_list[p];var
o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var
l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending
current state (see below).");console.debug(g);var
j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var
o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var
u=f.field_list[p];if(u.update){var
s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var
r in t.options){var
q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating
options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() -
Received new model (see
below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm()
- Refresh request failed.");console.debug(i)}})}});return{View:d}});
\ No newline at end of file
https://bitbucket.org/galaxy/galaxy-central/commits/eb6b7fb7a91c/
Changeset: eb6b7fb7a91c
User: guerler
Date: 2015-02-27 06:35:38+00:00
Summary: ToolForm/Collections: Re-introduce parameter expansion
Affected #: 4 files
diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r
eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 client/galaxy/scripts/mvc/tools/tools-form.js
--- a/client/galaxy/scripts/mvc/tools/tools-form.js
+++ b/client/galaxy/scripts/mvc/tools/tools-form.js
@@ -110,17 +110,6 @@
inputs : current_state
}
- // patch data tool parameters
- // TODO: This needs to be removed and handled in the api
- for (var i in current_state.inputs) {
- var dict = current_state.inputs[i];
- try {
- if (dict && dict.values[0].src === 'hda') {
- current_state.inputs[i] = self.content.get({id:
dict.values[0].id, src: 'hda'}).id_uncoded;
- }
- } catch (err) {}
- }
-
// activates/disables spinner for dynamic fields to indicate that they are
currently being updated
function wait(active) {
for (var i in self.input_list) {
diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r
eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 lib/galaxy/tools/__init__.py
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -2366,7 +2366,7 @@
else:
value, error = check_param(trans, input, default_value, context)
except Exception, err:
- log.error('Checking parameter failed. %s', str(err))
+ log.error('Checking parameter %s failed. %s', input.name,
str(err))
pass
return [value, error]
@@ -2482,6 +2482,11 @@
else:
state[k] = jsonify(state[k])
+ # expand incoming parameters (parameters might trigger multiple tool executions,
+ # here we select the first execution only in order to resolve dynamic
parameters)
+ expanded_incomings, _ = expand_meta_parameters( trans, self, params.__dict__ )
+ params.__dict__ = expanded_incomings[ 0 ]
+
# do param translation here, used by datasource tools
if self.input_translator:
self.input_translator.translate( params )
diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r
eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 static/scripts/mvc/tools/tools-form.js
--- a/static/scripts/mvc/tools/tools-form.js
+++ b/static/scripts/mvc/tools/tools-form.js
@@ -110,17 +110,6 @@
inputs : current_state
}
- // patch data tool parameters
- // TODO: This needs to be removed and handled in the api
- for (var i in current_state.inputs) {
- var dict = current_state.inputs[i];
- try {
- if (dict && dict.values[0].src === 'hda') {
- current_state.inputs[i] = self.content.get({id:
dict.values[0].id, src: 'hda'}).id_uncoded;
- }
- } catch (err) {}
- }
-
// activates/disables spinner for dynamic fields to indicate that they are
currently being updated
function wait(active) {
for (var i in self.input_list) {
diff -r e630e74d5571462111045b7db9b021427c7ab6ef -r
eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 static/scripts/packed/mvc/tools/tools-form.js
--- a/static/scripts/packed/mvc/tools/tools-form.js
+++ b/static/scripts/packed/mvc/tools/tools-form.js
@@ -1,1 +1,1 @@
-define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var
d=b.extend({initialize:function(g){var f=this;this.job_handler=new
a(this);this.buttons={execute:new
e.Button({icon:"fa-check",tooltip:"Execute:
"+g.name,title:"Execute",cls:"btn
btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var
f=this;var
g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var
i=top.location.href;var
j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var
h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now
you are using '"+f.options.name+"' version
"+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model
ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught
error.";f.modal.show({title:"Tool cannot be
executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var
f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};for(var
h in g.inputs){var
n=g.inputs[h];try{if(n&&n.values[0].src==="hda"){g.inputs[h]=f.content.get({id:n.values[0].id,src:"hda"}).id_uncoded}}catch(k){}}function
m(r){for(var p in f.input_list){var q=f.field_list[p];var
o=f.input_list[p];if(o.is_dynamic&&q.wait&&q.unwait){if(r){q.wait()}else{q.unwait()}}}}m(true);var
l=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending
current state (see below).");console.debug(g);var
j=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:j,data:g,success:function(i){f.tree.matchModel(i,function(p,t){var
o=f.input_list[p];if(o&&o.options){if(!_.isEqual(o.options,t.options)){o.options=t.options;var
u=f.field_list[p];if(u.update){var
s=[];if((["data","data_collection","drill_down"]).indexOf(o.type)!=-1){s=o.options}else{for(var
r in t.options){var
q=t.options[r];if(q.length>2){s.push({label:q[0],value:q[1]})}}}u.update(s);u.trigger("change");console.debug("Updating
options for "+p)}}}});m(false);console.debug("tools-form::_refreshForm() -
Received new model (see
below).");console.debug(i);f.deferred.done(l)},error:function(i){f.deferred.done(l);console.debug("tools-form::_refreshForm()
- Refresh request failed.");console.debug(i)}})}});return{View:d}});
\ No newline at end of file
+define(["utils/utils","mvc/ui/ui-misc","mvc/tools/tools-form-base","mvc/tools/tools-jobs"],function(c,e,b,a){var
d=b.extend({initialize:function(g){var f=this;this.job_handler=new
a(this);this.buttons={execute:new
e.Button({icon:"fa-check",tooltip:"Execute:
"+g.name,title:"Execute",cls:"btn
btn-primary",floating:"clear",onclick:function(){f.job_handler.submit()}})};g.initial_errors=g.job_id;b.prototype.initialize.call(this,g)},_buildModel:function(){var
f=this;var
g=galaxy_config.root+"api/tools/"+this.options.id+"/build?";if(this.options.job_id){g+="job_id="+this.options.job_id}else{if(this.options.dataset_id){g+="dataset_id="+this.options.dataset_id}else{g+="tool_version="+this.options.version+"&";var
i=top.location.href;var
j=i.indexOf("?");if(i.indexOf("tool_id=")!=-1&&j!==-1){g+=i.slice(j+1)}}}var
h=this.deferred.register();c.request({type:"GET",url:g,success:function(k){f.build(k);f.message.update({status:"success",message:"Now
you are using '"+f.options.name+"' version
"+f.options.version+".",persistent:false});f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model
ready.");console.debug(k)},error:function(k){f.deferred.done(h);console.debug("tools-form::initialize()
- Initial tool model request failed.");console.debug(k);var l=k.error||"Uncaught
error.";f.modal.show({title:"Tool cannot be
executed",body:l,buttons:{Close:function(){f.modal.hide()}}})}})},_updateModel:function(g){var
f=this;var g={tool_id:this.options.id,tool_version:this.options.version,inputs:g};function
j(n){for(var l in f.input_list){var m=f.field_list[l];var
k=f.input_list[l];if(k.is_dynamic&&m.wait&&m.unwait){if(n){m.wait()}else{m.unwait()}}}}j(true);var
i=this.deferred.register();console.debug("tools-form::_refreshForm() - Sending
current state (see below).");console.debug(g);var
h=galaxy_config.root+"api/tools/"+this.options.id+"/build";c.request({type:"POST",url:h,data:g,success:function(k){f.tree.matchModel(k,function(m,q){var
l=f.input_list[m];if(l&&l.options){if(!_.isEqual(l.options,q.options)){l.options=q.options;var
r=f.field_list[m];if(r.update){var
p=[];if((["data","data_collection","drill_down"]).indexOf(l.type)!=-1){p=l.options}else{for(var
o in q.options){var
n=q.options[o];if(n.length>2){p.push({label:n[0],value:n[1]})}}}r.update(p);r.trigger("change");console.debug("Updating
options for "+m)}}}});j(false);console.debug("tools-form::_refreshForm() -
Received new model (see
below).");console.debug(k);f.deferred.done(i)},error:function(k){f.deferred.done(i);console.debug("tools-form::_refreshForm()
- Refresh request failed.");console.debug(k)}})}});return{View:d}});
\ No newline at end of file
https://bitbucket.org/galaxy/galaxy-central/commits/46feb8f33ba9/
Changeset: 46feb8f33ba9
User: guerler
Date: 2015-02-27 06:49:53+00:00
Summary: Parameters: Prevent uncaught exception on unavailable datasets in column
validation
Affected #: 1 file
diff -r eb6b7fb7a91cc4e344bcf23ba23ff14807e11ce5 -r
46feb8f33ba97dc193fb9cecd9025c18803e2307 lib/galaxy/tools/parameters/basic.py
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -1340,6 +1340,10 @@
datasets = util.listify( referent )
for dataset in datasets:
if dataset:
+ # Check if metadata is available
+ if not hasattr(dataset, 'metadata'):
+ return True
+
# Check if the dataset does not have the expected metadata for columns
if not dataset.metadata.columns:
# Only allow late validation if the dataset is not yet ready
https://bitbucket.org/galaxy/galaxy-central/commits/448272e06e42/
Changeset: 448272e06e42
User: guerler
Date: 2015-02-27 06:57:39+00:00
Summary: ToolForm/Ui: Limit selection to available options
Affected #: 3 files
diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r
448272e06e42db47dec46a3a0dae58f9ca265160
client/galaxy/scripts/mvc/ui/ui-select-default.js
--- a/client/galaxy/scripts/mvc/ui/ui-select-default.js
+++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js
@@ -78,9 +78,11 @@
if (new_value === null) {
new_value = '__null__';
}
- this.$select.val(new_value);
- if (this.$select.select2) {
- this.$select.select2('val', new_value);
+ if (this.exists(new_value)) {
+ this.$select.val(new_value);
+ if (this.$select.select2) {
+ this.$select.select2('val', new_value);
+ }
}
}
diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r
448272e06e42db47dec46a3a0dae58f9ca265160 static/scripts/mvc/ui/ui-select-default.js
--- a/static/scripts/mvc/ui/ui-select-default.js
+++ b/static/scripts/mvc/ui/ui-select-default.js
@@ -78,9 +78,11 @@
if (new_value === null) {
new_value = '__null__';
}
- this.$select.val(new_value);
- if (this.$select.select2) {
- this.$select.select2('val', new_value);
+ if (this.exists(new_value)) {
+ this.$select.val(new_value);
+ if (this.$select.select2) {
+ this.$select.select2('val', new_value);
+ }
}
}
diff -r 46feb8f33ba97dc193fb9cecd9025c18803e2307 -r
448272e06e42db47dec46a3a0dae58f9ca265160
static/scripts/packed/mvc/ui/ui-select-default.js
--- a/static/scripts/packed/mvc/ui/ui-select-default.js
+++ b/static/scripts/packed/mvc/ui/ui-select-default.js
@@ -1,1 +1,1 @@
-define(["utils/utils"],function(a){var
b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No
data available",empty_text:"No
selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var
c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}return
this._getValue()},first:function(){var
c=this.$select.find("option").first();if(c.length>0){return
c.val()}else{return null}},text:function(){return
this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-spinner
fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-caret-down")},disabled:function(){return
this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var
e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var
d in
c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return
this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var
c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var
c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option
value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div
id="'+c.id+'" class="'+c.cls+'"><select
id="'+c.id+'_select" class="select"/><div
class="button"><i
class="icon"/></div></div>'}});return{View:b}});
\ No newline at end of file
+define(["utils/utils"],function(a){var
b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No
data available",empty_text:"No
selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var
c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return
this._getValue()},first:function(){var
c=this.$select.find("option").first();if(c.length>0){return
c.val()}else{return null}},text:function(){return
this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-spinner
fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-caret-down")},disabled:function(){return
this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var
e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var
d in
c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return
this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var
c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var
c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option
value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div
id="'+c.id+'" class="'+c.cls+'"><select
id="'+c.id+'_select" class="select"/><div
class="button"><i
class="icon"/></div></div>'}});return{View:b}});
\ No newline at end of file
https://bitbucket.org/galaxy/galaxy-central/commits/94298235c5c4/
Changeset: 94298235c5c4
User: guerler
Date: 2015-02-27 06:59:43+00:00
Summary: Parameters: Fix deleted datasets validation message
Affected #: 1 file
diff -r 448272e06e42db47dec46a3a0dae58f9ca265160 -r
94298235c5c4df5f53bbfc5f7bb742837781001c lib/galaxy/tools/parameters/basic.py
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -2021,7 +2021,7 @@
for v in values:
if v:
if v.deleted:
- raise ValueError( "The previously selected dataset has been
previously deleted" )
+ raise ValueError( "The previously selected dataset has been
deleted." )
if hasattr( v, "dataset" ) and v.dataset.state in [
galaxy.model.Dataset.states.ERROR, galaxy.model.Dataset.states.DISCARDED ]:
raise ValueError( "The previously selected dataset has entered
an unusable state" )
if not self.multiple:
https://bitbucket.org/galaxy/galaxy-central/commits/81c09bf641da/
Changeset: 81c09bf641da
User: guerler
Date: 2015-02-27 09:26:15+00:00
Summary: Parameters: Set hidden attribute for hidden dataset parameter
Affected #: 1 file
diff -r 94298235c5c4df5f53bbfc5f7bb742837781001c -r
81c09bf641da8461655ded2b8638fe2c203aa0b1 lib/galaxy/tools/parameters/basic.py
--- a/lib/galaxy/tools/parameters/basic.py
+++ b/lib/galaxy/tools/parameters/basic.py
@@ -2425,6 +2425,7 @@
DataToolParameter.__init__( self, tool, elem )
self.value = "None"
self.type = "hidden_data"
+ self.hidden = True
def get_initial_value( self, trans, context, history=None ):
return None
https://bitbucket.org/galaxy/galaxy-central/commits/b11c4470b151/
Changeset: b11c4470b151
User: guerler
Date: 2015-02-27 09:36:20+00:00
Summary: ToolForm/Ui: Restrict available option requirement to single select fields
Affected #: 3 files
diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r
b11c4470b151100c14f1a2abfb8047ac16b26f30
client/galaxy/scripts/mvc/ui/ui-select-default.js
--- a/client/galaxy/scripts/mvc/ui/ui-select-default.js
+++ b/client/galaxy/scripts/mvc/ui/ui-select-default.js
@@ -78,7 +78,7 @@
if (new_value === null) {
new_value = '__null__';
}
- if (this.exists(new_value)) {
+ if (this.exists(new_value) || this.options.multiple) {
this.$select.val(new_value);
if (this.$select.select2) {
this.$select.select2('val', new_value);
diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r
b11c4470b151100c14f1a2abfb8047ac16b26f30 static/scripts/mvc/ui/ui-select-default.js
--- a/static/scripts/mvc/ui/ui-select-default.js
+++ b/static/scripts/mvc/ui/ui-select-default.js
@@ -78,7 +78,7 @@
if (new_value === null) {
new_value = '__null__';
}
- if (this.exists(new_value)) {
+ if (this.exists(new_value) || this.options.multiple) {
this.$select.val(new_value);
if (this.$select.select2) {
this.$select.select2('val', new_value);
diff -r 81c09bf641da8461655ded2b8638fe2c203aa0b1 -r
b11c4470b151100c14f1a2abfb8047ac16b26f30
static/scripts/packed/mvc/ui/ui-select-default.js
--- a/static/scripts/packed/mvc/ui/ui-select-default.js
+++ b/static/scripts/packed/mvc/ui/ui-select-default.js
@@ -1,1 +1,1 @@
-define(["utils/utils"],function(a){var
b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No
data available",empty_text:"No
selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var
c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return
this._getValue()},first:function(){var
c=this.$select.find("option").first();if(c.length>0){return
c.val()}else{return null}},text:function(){return
this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-spinner
fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-caret-down")},disabled:function(){return
this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var
e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var
d in
c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return
this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var
c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var
c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option
value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div
id="'+c.id+'" class="'+c.cls+'"><select
id="'+c.id+'_select" class="select"/><div
class="button"><i
class="icon"/></div></div>'}});return{View:b}});
\ No newline at end of file
+define(["utils/utils"],function(a){var
b=Backbone.View.extend({optionsDefault:{id:"",cls:"ui-select",error_text:"No
data available",empty_text:"No
selection",visible:true,wait:false,multiple:false,searchable:true,optional:false},initialize:function(d){this.options=a.merge(d,this.optionsDefault);this.setElement(this._template(this.options));this.$select=this.$el.find(".select");this.$icon=this.$el.find(".icon");this.$button=this.$el.find(".button");if(this.options.multiple){this.$el.addClass("ui-select-multiple");this.$select.prop("multiple",true);this.$button.remove()}this.update(this.options.data);if(this.options.value!==undefined){this.value(this.options.value)}if(!this.options.visible){this.hide()}if(this.options.wait){this.wait()}else{this.show()}var
c=this;this.$select.on("change",function(){c._change()});this.on("change",function(){c._change()})},value:function(c){if(c!==undefined){if(c===null){c="__null__"}if(this.exists(c)||this.options.multiple){this.$select.val(c);if(this.$select.select2){this.$select.select2("val",c)}}}return
this._getValue()},first:function(){var
c=this.$select.find("option").first();if(c.length>0){return
c.val()}else{return null}},text:function(){return
this.$select.find("option:selected").text()},show:function(){this.unwait();this.$select.show();this.$el.show()},hide:function(){this.$el.hide()},wait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-spinner
fa-spin")},unwait:function(){this.$icon.removeClass();this.$icon.addClass("fa
fa-caret-down")},disabled:function(){return
this.$select.is(":disabled")},enable:function(){this.$select.prop("disabled",false)},disable:function(){this.$select.prop("disabled",true)},add:function(c){this.$select.append(this._templateOption(c));this._refresh()},del:function(c){this.$select.find("option[value="+c+"]").remove();this.$select.trigger("change");this._refresh()},update:function(c){var
e=this._getValue();this.$select.find("option").remove();if(!this.options.multiple&&this.options.optional){this.$select.append(this._templateOption({value:"__null__",label:this.options.empty_text}))}for(var
d in
c){this.$select.append(this._templateOption(c[d]))}this._refresh();if(this.options.searchable){this.$select.select2("destroy");this.$select.select2()}this.value(e);if(this._getValue()===null){this.value(this.first())}},setOnChange:function(c){this.options.onchange=c},exists:function(c){return
this.$select.find('option[value="'+c+'"]').length>0},_change:function(){if(this.options.onchange){this.options.onchange(this._getValue())}},_getValue:function(){var
c=this.$select.val();if(!a.validate(c)){return null}return c},_refresh:function(){var
c=this.$select.find("option").length;if(c==0){this.disable();this.$select.empty();this.$select.append(this._templateOption({value:"__null__",label:this.options.error_text}))}else{this.enable()}},_templateOption:function(c){return'<option
value="'+c.value+'">'+c.label+"</option>"},_template:function(c){return'<div
id="'+c.id+'" class="'+c.cls+'"><select
id="'+c.id+'_select" class="select"/><div
class="button"><i
class="icon"/></div></div>'}});return{View:b}});
\ No newline at end of file
https://bitbucket.org/galaxy/galaxy-central/commits/6bd6c350296c/
Changeset: 6bd6c350296c
User: carlfeberhard
Date: 2015-02-27 15:03:47+00:00
Summary: Fix display_structure by using serializer; fix structure by removing extra
decode_id
Affected #: 2 files
diff -r b11c4470b151100c14f1a2abfb8047ac16b26f30 -r
6bd6c350296c94817de06d3c239b9bf8d2b9978c lib/galaxy/webapps/galaxy/controllers/history.py
--- a/lib/galaxy/webapps/galaxy/controllers/history.py
+++ b/lib/galaxy/webapps/galaxy/controllers/history.py
@@ -543,8 +543,10 @@
def structure( self, trans, id=None, **kwargs ):
"""
"""
- id = self.decode_id( id ) if id else trans.history.id
- history_to_view = self.history_manager.get_accessible( trans, id, trans.user )
+ unencoded_history_id = trans.history.id
+ if id:
+ unencoded_history_id = self.decode_id( id )
+ history_to_view = self.history_manager.get_accessible( trans,
unencoded_history_id, trans.user )
history_data = self.history_manager._get_history_data( trans, history_to_view )
history_dictionary = history_data[ 'history' ]
@@ -552,7 +554,7 @@
jobs = ( trans.sa_session.query( trans.app.model.Job )
.filter( trans.app.model.Job.user == trans.user )
- .filter( trans.app.model.Job.history_id == self.decode_id( id ) ) ).all()
+ .filter( trans.app.model.Job.history_id == unencoded_history_id ) ).all()
jobs = map( lambda j: self.encode_all_ids( trans, j.to_dict( 'element' ),
True ), jobs )
tools = {}
diff -r b11c4470b151100c14f1a2abfb8047ac16b26f30 -r
6bd6c350296c94817de06d3c239b9bf8d2b9978c
templates/webapps/galaxy/history/display_structured.mako
--- a/templates/webapps/galaxy/history/display_structured.mako
+++ b/templates/webapps/galaxy/history/display_structured.mako
@@ -128,13 +128,15 @@
## ----------------------------------------------------------------------------
<%def name="javascripts()"><%
+ from galaxy.managers import hdas
+ hda_serializer = hdas.HDASerializer( trans.app )
self.js_app = 'display-structured'
controller = trans.webapp.controllers[ 'history' ]
hda_dicts = []
id_hda_dict_map = {}
for hda in history.active_datasets:
- hda_dict = controller.get_hda_dict( trans, hda )
+ hda_dict = hda_serializer.serialize_to_view( trans, hda, 'detailed' )
id_hda_dict_map[ hda_dict[ 'id' ] ] = hda_dict
hda_dicts.append( hda_dict )
%>
Repository URL:
https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from
bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.