1 new commit in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/70f7ff8c6a96/ Changeset: 70f7ff8c6a96 User: jmchilton Date: 2014-10-20 21:22:27+00:00 Summary: Merged in jmchilton/galaxy-central-ie (pull request #533) Interactive Environments Plugin Framework with IPython Plugin Affected #: 35 files diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 client/galaxy/scripts/galaxy.interactive_environments.js --- /dev/null +++ b/client/galaxy/scripts/galaxy.interactive_environments.js @@ -0,0 +1,59 @@ +/** + * Internal function to remove content from the main area and add the notebook. + * Not idempotent + */ +function append_notebook(url){ + clear_main_area(); + $('#main').append('<iframe frameBorder="0" seamless="seamless" style="width: 100%; height: 100%; overflow:hidden;" scrolling="no" src="'+ url +'"></iframe>' + ); +} + +function clear_main_area(){ + $('#spinner').remove(); + $('#main').children().remove(); +} + +function display_spinner(){ + $('#main').append('<img id="spinner" src="' + galaxy_root + '/static/style/largespinner.gif" style="position:absolute;margin:auto;top:0;left:0;right:0;bottom:0;">'); +} + + +/** + * Test availability of a URL, and call a callback when done. + * http://stackoverflow.com/q/25390206/347368 + * @param {String} url: URL to test availability of. Must return a 200 (302->200 is OK). + * @param {String} callback: function to call once successfully connected. + * + */ +function test_ie_availability(url, success_callback){ + var request_count = 0; + display_spinner(); + interval = setInterval(function(){ + $.ajax({ + url: url, + xhrFields: { + withCredentials: true + }, + type: "GET", + timeout: 500, + success: function(){ + console.log("Connected to IE, returning"); + clearInterval(interval); + success_callback(); + }, + error: function(jqxhr, status, error){ + request_count++; + console.log("Request " + request_count); + if(request_count > 30){ + clearInterval(interval); + clear_main_area(); + toastr.error( + "Could not connect to IE, contact your administrator", + "Error", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} + ); + } + } + }); + }, 1000); +} diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/datatypes_conf.xml.sample --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -249,6 +249,8 @@ <datatype extension="snpmatrix" type="galaxy.datatypes.genetics:SNPMatrix" display_in_upload="true"/><datatype extension="xls" type="galaxy.datatypes.tabular:Tabular"/><!-- End RGenetics Datatypes --> + <datatype extension="ipynb" type="galaxy.datatypes.text:Ipynb" display_in_upload="True" /> + <datatype extension="json" type="galaxy.datatypes.text:Json" display_in_upload="True" /><!-- graph datatypes --><datatype extension="xgmml" type="galaxy.datatypes.graph:Xgmml" display_in_upload="true"/><datatype extension="sif" type="galaxy.datatypes.graph:Sif" display_in_upload="true"/> @@ -292,6 +294,8 @@ <sniffer type="galaxy.datatypes.tabular:Sam"/><sniffer type="galaxy.datatypes.data:Newick"/><sniffer type="galaxy.datatypes.data:Nexus"/> + <sniffer type="galaxy.datatypes.text:Ipynb"/> + <sniffer type="galaxy.datatypes.text:Json"/><sniffer type="galaxy.datatypes.images:Jpg"/><sniffer type="galaxy.datatypes.images:Png"/><sniffer type="galaxy.datatypes.images:Tiff"/> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/galaxy.ini.sample --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -221,9 +221,15 @@ # Visualizations config directory: where to look for individual visualization plugins. # The path is relative to the Galaxy root dir. To use an absolute path begin the path -# with '/'. Defaults to "config/plugins/visualizations". +# with '/'. This is a comma separated list. Defaults to "config/plugins/visualizations". #visualization_plugins_directory = config/plugins/visualizations +# Interactive environment plugins root directory: where to look for interactive environment +# plugins. By default none will be loaded. Set to config/plugins/interactive_environments +# to load Galaxy's stock plugins (currently just IPython). These will require Docker +# to be configured and have security considerations so proceed with caution. +#interactive_environment_plugins_directory = + # Each job is given a unique empty directory as its current working directory. # This option defines in what parent directory those directories will be # created. @@ -421,6 +427,11 @@ # - $iso8601 (complete format string as specified by ISO 8601 international standard). # pretty_datetime_format = $locale (UTC) +# URL (with schema http/https) of the Galaxy instance as accessible within your local +# network - if specified used as a default by pulsar file staging and IPython docker +# container for communicating back with Galaxy via the API. +#galaxy_infrastructure_url = http://localhost:8080 + # The URL of the page to display in Galaxy's middle pane when loaded. This can be # an absolute or relative URL. #welcome_url = /static/welcome.html @@ -517,6 +528,26 @@ # requests. #nginx_upload_path = False +# Have Galaxy manage dynamic proxy component for routing requests to other +# services based on Galaxy's session cookie. It will attempt to do this by +# default though you do need to install node+npm and do an npm install from +# `lib/galaxy/web/proxy/js`. It is generally more robust to configure this +# externally managing it however Galaxy is managed. If True Galaxy will only +# launch the proxy if it is actually going to be used (e.g. for IPython). +#dynamic_proxy_manage=True + +# Dynamic proxy can use an SQLite database or a JSON file for IPC, set that +# here. +#dynamic_proxy_session_map=database/session_map.sqlite + +# Set the port and IP for the the dynamic proxy to bind to, this must match +# the external configuration if dynamic_proxy_manage is False. +#dynamic_proxy_bind_port=8800 +#dynamic_proxy_bind_ip=0.0.0.0 + +# Enable verbose debugging of Galaxy-managed dynamic proxy. +#dynamic_proxy_debug=False + # -- Logging and Debugging # Verbosity of console log messages. Acceptable values can be found here: @@ -549,6 +580,12 @@ # by setting the following option to True. #serve_xss_vulnerable_mimetypes = False +# Set the following to True to use ipython nbconvert to build HTML from IPython +# notebooks in Galaxy histories. This process may allow users to execute arbitrary +# code or serve arbitrary HTML. If enabled ipython must be available and on Galaxy's +# PATH, to do this run `pip install jinja2 pygments ipython` in Galaxy's virtualenv. +#trust_ipython_notebook_conversion = False + # Debug enables access to various config options useful for development and # debugging: use_lint, use_profile, use_printdebug and use_interactive. It # also causes the files used by PBS/SGE (submission script, output, and error) diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/job_conf.xml.sample_advanced --- a/config/job_conf.xml.sample_advanced +++ b/config/job_conf.xml.sample_advanced @@ -50,7 +50,9 @@ <!-- AMQP URL to connect to. --><param id="amqp_url">amqp://guest:guest@localhost:5672//</param><!-- URL remote Pulsar apps should transfer files to this Galaxy - instance to/from. --> + instance to/from. This can be unspecified/empty if + galaxy_infrastructure_url is set in galaxy.ini. + --><param id="galaxy_url">http://localhost:8080</param><!-- Pulsar job manager to communicate with (see Pulsar docs for information on job managers). --> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/common/templates/ie.mako --- /dev/null +++ b/config/plugins/interactive_environments/common/templates/ie.mako @@ -0,0 +1,31 @@ +<%def name="default_javascript_variables()"> +// Globals + +// Following three are for older-style IE proxies, newer dynamic Galaxy proxy +// does not use these. +ie_password_auth = ${ ie_request.javascript_boolean(ie_request.attr.PASSWORD_AUTH) }; +ie_apache_urls = ${ ie_request.javascript_boolean(ie_request.attr.APACHE_URLS) }; +ie_password = '${ ie_request.attr.notebook_pw }'; + + +var galaxy_root = '${ ie_request.attr.root }'; +var app_root = '${ ie_request.attr.app_root }'; +</%def> + + +<%def name="load_default_js()"> +${h.css( 'base' ) } +${h.js( 'libs/jquery/jquery', + 'libs/toastr', + 'libs/require')} +</%def> + +<%def name="plugin_require_config()"> +require.config({ + baseUrl: app_root, + paths: { + "plugin" : app_root + "js/", + "interactive_environments": "${h.url_for('/static/scripts/galaxy.interactive_environments')}", + }, +}); +</%def> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/interactive_environments.dtd --- /dev/null +++ b/config/plugins/interactive_environments/interactive_environments.dtd @@ -0,0 +1,1 @@ +../visualizations/visualization.dtd \ No newline at end of file diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/ipython/config/ipython.ini.sample --- /dev/null +++ b/config/plugins/interactive_environments/ipython/config/ipython.ini.sample @@ -0,0 +1,16 @@ +[main] + +# Following options are ignored if using the Galaxy dynamic proxy but +# are useful if mapping a range of ports for environment consumption. +#apache_urls = False +#password_auth = False +#ssl = False + +[docker] +command = docker +image = bgruening/docker-ipython-notebook + +# URL to access the Galaxy API with from the spawn Docker containter, if empty +# this falls back to galaxy.ini's galaxy_infrastructure_url and finally to the +# Docker host of the spawned container if that is also not set. +#galaxy_url = diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/ipython/config/ipython.xml --- /dev/null +++ b/config/plugins/interactive_environments/ipython/config/ipython.xml @@ -0,0 +1,16 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE interactive_environment SYSTEM "../../interactive_environments.dtd"> +<interactive_environment name="IPython"> + <data_sources> + <data_source> + <model_class>HistoryDatasetAssociation</model_class> + <test type="isinstance" test_attr="datatype" result_type="datatype">tabular.Tabular</test> + <test type="isinstance" test_attr="datatype" result_type="datatype">data.Text</test> + <to_param param_attr="id">dataset_id</to_param> + </data_source> + </data_sources> + <params> + <param type="dataset" var_name_in_template="hda" required="true">dataset_id</param> + </params> + <template>ipython.mako</template> +</interactive_environment> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/ipython/static/js/ipython.js --- /dev/null +++ b/config/plugins/interactive_environments/ipython/static/js/ipython.js @@ -0,0 +1,85 @@ +function message_failed_auth(password){ + toastr.info( + "Automatic authorization failed. You can manually login with:<br>" + password + "<br><a href='https://github.com/bgruening/galaxy-ipython/wiki/Automatic-Authorization-Fai...' target='_blank'>More details ...</a>", + "Please login manually", + {'closeButton': true, 'timeOut': 100000, 'tapToDismiss': false} + ); +} + +function message_failed_connection(){ + toastr.error( + "Could not connect to IPython Notebook. Please contact your administrator. <a href='https://github.com/bgruening/galaxy-ipython/wiki/Could-not-connect-to-IPytho...' target='_blank'>More details ...</a>", + "Security warning", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': true} + ); +} + +function message_no_auth(){ + // No longer a security issue, proxy validates Galaxy session token. + /* + toastr.warning( + "IPython Notebook was lunched without authentication. This is a security issue. <a href='https://github.com/bgruening/galaxy-ipython/wiki/IPython-Notebook-was-lunche...' target='_blank'>More details ...</a>", + "Security warning", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} + ); + */ +} + + +/** + * Load an interactive environment (IE) from a remote URL + * @param {String} password: password used to authenticate to the remote resource + * @param {String} notebook_login_url: URL that should be POSTed to for login + * @param {String} notebook_access_url: the URL embeded in the page and loaded + * + */ +function load_notebook(password, notebook_login_url, notebook_access_url){ + $( document ).ready(function() { + // Test notebook_login_url for accessibility, executing the login+load function whenever + // we've successfully connected to the IE. + test_ie_availability(notebook_login_url, function(){ + _handle_notebook_loading(password, notebook_login_url, notebook_access_url); + }); + }); +} + +/** + * Must be implemented by IEs + */ +function _handle_notebook_loading(password, notebook_login_url, notebook_access_url){ + if ( ie_password_auth ) { + // Make an AJAX POST + $.ajax({ + type: "POST", + // to the Login URL + url: notebook_login_url, + // With our password + data: { + 'password': password + }, + xhrFields: { + withCredentials: true + }, + // If that is successful, load the notebook + success: function(){ + append_notebook(notebook_access_url); + }, + error: function(jqxhr, status, error){ + if(ie_password_auth && !ie_apache_urls){ + // Failure happens due to CORS + message_failed_auth(password); + append_notebook(notebook_access_url); + }else{ + message_failed_connection(); + // Do we want to try and load the notebook anyway? Just in case? + append_notebook(notebook_access_url); + } + } + }); + } + else { + // Not using password auth, just embed it to avoid content-origin issues. + message_no_auth(); + append_notebook(notebook_access_url); + } +} diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/ipython/templates/ipython.mako --- /dev/null +++ b/config/plugins/interactive_environments/ipython/templates/ipython.mako @@ -0,0 +1,61 @@ + <%namespace name="ie" file="ie.mako" /> + +<% +import os +import shutil +import tempfile +import subprocess + +# Sets ID and sets up a lot of other variables +ie_request.load_deploy_config() +ie_request.attr.docker_port = 6789 +# Create tempdir in galaxy +temp_dir = os.path.abspath( tempfile.mkdtemp() ) +# Write out conf file...needs work +ie_request.write_conf_file(temp_dir) + +## IPython Specific +# Prepare an empty notebook +notebook_id = ie_request.generate_hex(64) +with open( os.path.join( ie_request.attr.our_template_dir, 'notebook.ipynb' ), 'r') as nb_handle: + empty_nb = nb_handle.read() +empty_nb = empty_nb % notebook_id +# Copy over default notebook, unless the dataset this viz is running on is a notebook +empty_nb_path = os.path.join(temp_dir, 'ipython_galaxy_notebook.ipynb') +if hda.datatype.__class__.__name__ != "Ipynb": + with open( empty_nb_path, 'w+' ) as handle: + handle.write( empty_nb ) +else: + shutil.copy( hda.file_name, empty_nb_path ) + + +## General IE specific +# Access URLs for the notebook from within galaxy. +notebook_access_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/notebooks/ipython_galaxy_notebook.ipynb') +notebook_login_url = ie_request.url_template('${PROXY_URL}/ipython/${PORT}/login?next=%2Fipython%2F${PORT}%2Ftree') + +docker_cmd = ie_request.docker_cmd(temp_dir) +subprocess.call(docker_cmd, shell=True) +%> +<html> +<head> +${ ie.load_default_js() } +</head> +<body> + +<script type="text/javascript"> +${ ie.default_javascript_variables() } +var notebook_login_url = '${ notebook_login_url }'; +var notebook_access_url = '${ notebook_access_url }'; +${ ie.plugin_require_config() } + +// Load notebook + +requirejs(['interactive_environments', 'plugin/ipython'], function(){ + load_notebook(ie_password, notebook_login_url, notebook_access_url); +}); +</script> +<div id="main" width="100%" height="100%"> +</div> +</body> +</html> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/interactive_environments/ipython/templates/notebook.ipynb --- /dev/null +++ b/config/plugins/interactive_environments/ipython/templates/notebook.ipynb @@ -0,0 +1,43 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:%s" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Welcome to the interactive Galaxy IPython Notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can access your data via the dataset number. For example, ``handle = get(42)``.", + "To save data, write your data to a file, and then call ``put('filename.txt')``. The dataset will then be available in your galaxy history.", + "Notebooks can be saved to Galaxy by clicking the large green button at the top right of the IPython interface.<br>", + "More help and informations can be found on the project [website](https://github.com/bgruening/galaxy-ipython)." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + } + ], + "metadata": {} + } + ] +} diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/visualizations/additional_template_paths.xml --- a/config/plugins/visualizations/additional_template_paths.xml +++ b/config/plugins/visualizations/additional_template_paths.xml @@ -2,4 +2,5 @@ <!-- these relative paths can contain common templates importable by all visualization plugins --><paths><path>common/templates</path> + <path>../interactive_environments/common/templates</path></paths> diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 config/plugins/visualizations/visualization.dtd --- a/config/plugins/visualizations/visualization.dtd +++ b/config/plugins/visualizations/visualization.dtd @@ -1,5 +1,6 @@ <!-- each visualization must have a template (all other elements are optional) --><!ELEMENT visualization (description*,data_sources*,params*,template_root*,template,render_target*)> +<!ELEMENT interactive_environment (description*,data_sources*,params*,template_root*,template,render_target*)><!-- visualization name: the title/display name of the visualization (e.g. 'Trackster', 'Fastq Stats', etc.) REQUIRED disabled: if included (value does not matter), this attribute will prevent the visualization being loaded diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/app.py --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -17,6 +17,7 @@ from galaxy.tools.data_manager.manager import DataManagers from galaxy.jobs import metrics as job_metrics from galaxy.web.base import pluginframework +from galaxy.web.proxy import ProxyManager from galaxy.queue_worker import GalaxyQueueWorker from tool_shed.galaxy_install import update_repository_manager @@ -138,6 +139,7 @@ # FIXME: These are exposed directly for backward compatibility self.job_queue = self.job_manager.job_queue self.job_stop_queue = self.job_manager.job_stop_queue + self.proxy_manager = ProxyManager( self.config ) # Initialize the external service types self.external_service_types = external_service_types.ExternalServiceTypesCollection( self.config.external_service_type_config_file, self.config.external_service_type_path, self ) self.model.engine.dispose() diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/config.py --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -18,6 +18,8 @@ from galaxy.util.dbkeys import GenomeBuilds from galaxy import eggs +import ConfigParser + log = logging.getLogger( __name__ ) @@ -208,6 +210,7 @@ self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) ) self.sanitize_all_html = string_as_bool( kwargs.get( 'sanitize_all_html', True ) ) self.serve_xss_vulnerable_mimetypes = string_as_bool( kwargs.get( 'serve_xss_vulnerable_mimetypes', False ) ) + self.trust_ipython_notebook_conversion = string_as_bool( kwargs.get( 'trust_ipython_notebook_conversion', False ) ) self.enable_old_display_applications = string_as_bool( kwargs.get( "enable_old_display_applications", "True" ) ) self.brand = kwargs.get( 'brand', None ) self.welcome_url = kwargs.get( 'welcome_url', '/static/welcome.html' ) @@ -296,6 +299,23 @@ for section in global_conf_parser.sections(): if section.startswith('server:'): self.server_names.append(section.replace('server:', '', 1)) + + # Default URL (with schema http/https) of the Galaxy instance within the + # local network - used to remotely communicate with the Galaxy API. + galaxy_infrastructure_url = kwargs.get( 'galaxy_infrastructure_url', None ) + galaxy_infrastructure_url_set = True + if galaxy_infrastructure_url is None: + # Still provide a default but indicate it was not explicitly set + # so dependending on the context a better default can be used ( + # request url in a web thread, Docker parent in IE stuff, etc...) + galaxy_infrastructure_url = "http://localhost" + port = self.guess_galaxy_port() + if port: + galaxy_infrastructure_url += ":%s" % (port) + galaxy_infrastructure_url_set = False + self.galaxy_infrastructure_url = galaxy_infrastructure_url + self.galaxy_infrastructure_url_set = galaxy_infrastructure_url_set + # Store advanced job management config self.job_manager = kwargs.get('job_manager', self.server_name).strip() self.job_handlers = [ x.strip() for x in kwargs.get('job_handlers', self.server_name).split(',') ] @@ -357,6 +377,18 @@ # directory where the visualization/registry searches for plugins self.visualization_plugins_directory = kwargs.get( 'visualization_plugins_directory', 'config/plugins/visualizations' ) + ie_dirs = kwargs.get( 'interactive_environment_plugins_directory', None ) + if ie_dirs and not self.visualization_plugins_directory: + self.visualization_plugins_directory = ie_dirs + elif ie_dirs: + self.visualization_plugins_directory += ",%s" % ie_dirs + + self.proxy_session_map = self.resolve_path( kwargs.get( "dynamic_proxy_session_map", "database/session_map.sqlite" ) ) + self.manage_dynamic_proxy = string_as_bool( kwargs.get( "dynamic_proxy_manage", "True" ) ) # Set to false if being launched externally + self.dynamic_proxy_debug = string_as_bool( kwargs.get( "dynamic_proxy_debug", "False" ) ) + self.dynamic_proxy_bind_port = int( kwargs.get( "dynamic_proxy_bind_port", "8800" ) ) + self.dynamic_proxy_bind_ip = kwargs.get( "dynamic_proxy_bind_ip", "0.0.0.0" ) + # Default chunk size for chunkable datatypes -- 64k self.display_chunk_size = int( kwargs.get( 'display_chunk_size', 65536) ) @@ -543,6 +575,19 @@ """ return resolve_path( path, self.root ) + def guess_galaxy_port(self): + # Code derived from IPython work ie.mako + config = ConfigParser.SafeConfigParser({'port': '8080'}) + if self.config_file: + config.read( self.config_file ) + + try: + port = config.getint('server:%s' % self.server_name, 'port') + except: + # uWSGI galaxy installations don't use paster and only speak uWSGI not http + port = None + return port + def get_database_engine_options( kwargs, model_prefix='' ): """ diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/datatypes/registry.py --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -22,6 +22,7 @@ import assembly import ngsindex import graph +import text import galaxy.util from galaxy.util.odict import odict from display_applications.application import DisplayApplication diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/datatypes/text.py --- /dev/null +++ b/lib/galaxy/datatypes/text.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" Clearing house for generic text datatypes that are not XML or tabular. +""" + + +from galaxy.datatypes.data import Text +from galaxy.datatypes.data import get_file_peek +from galaxy.datatypes.data import nice_size +from galaxy import util + +import tempfile +import subprocess +import json +import os + +import logging +log = logging.getLogger(__name__) + + +class Json( Text ): + file_ext = "json" + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "JavaScript Object Notation (JSON)" + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to load the string with the json module. If successful it's a json file. + """ + return self._looks_like_json( filename ) + + def _looks_like_json( self, filename ): + # Pattern used by SequenceSplitLocations + if os.path.getsize(filename) < 50000: + # If the file is small enough - don't guess just check. + try: + json.load( open(filename, "r") ) + return True + except Exception: + return False + else: + with open(filename, "r") as fh: + while True: + line = fh.readline() + line = line.strip() + if line: + # simple types are valid JSON as well - but would such a file + # be interesting as JSON in Galaxy? + return line.startswith("[") or line.startswith("{") + return False + + def display_peek( self, dataset ): + try: + return dataset.peek + except: + return "JSON file (%s)" % ( nice_size( dataset.get_size() ) ) + + +class Ipynb( Json ): + file_ext = "ipynb" + + def set_peek( self, dataset, is_multi_byte=False ): + if not dataset.dataset.purged: + dataset.peek = get_file_peek( dataset.file_name, is_multi_byte=is_multi_byte ) + dataset.blurb = "IPython Notebook" + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disc' + + def sniff( self, filename ): + """ + Try to load the string with the json module. If successful it's a json file. + """ + if self._looks_like_json( filename ): + try: + ipynb = json.load( open(filename) ) + if ipynb.get('nbformat', False) is not False and ipynb.get('metadata', False): + return True + else: + return False + except: + return False + + def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): + config = trans.app.config + trust = getattr( config, 'trust_ipython_notebook_conversion', False ) + if trust: + return self._display_data_trusted(trans, dataset, preview=preview, fileame=filename, to_ext=to_ext, chunk=chunk, **kwd) + else: + return super(Ipynb, self).display_data( trans, dataset, preview=preview, fileame=filename, to_ext=to_ext, chunk=chunk, **kwd ) + + def _display_data_trusted(self, trans, dataset, preview=False, filename=None, to_ext=None, chunk=None, **kwd): + preview = util.string_as_bool( preview ) + if chunk: + return self.get_chunk(trans, dataset, chunk) + elif to_ext or not preview: + return self._serve_raw(trans, dataset, to_ext) + else: + ofile_handle = tempfile.NamedTemporaryFile(delete=False) + ofilename = ofile_handle.name + ofile_handle.close() + try: + cmd = 'ipython nbconvert --to html --template basic %s --output %s' % (dataset.file_name, ofilename) + log.info("Calling command %s" % cmd) + subprocess.call(cmd, shell=True) + ofilename = '%s.html' % ofilename + except: + ofilename = dataset.file_name + log.exception( 'Command "%s" failed. Could not convert the IPython Notebook to HTML, defaulting to plain text.' % cmd ) + return open( ofilename ) + + def set_meta( self, dataset, **kwd ): + """ + Set the number of models in dataset. + """ + pass diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/jobs/runners/pulsar.py --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -53,7 +53,7 @@ ), galaxy_url=dict( map=specs.to_str_or_none, - default=DEFAULT_GALAXY_URL, + default=None, ), manager=dict( map=specs.to_str_or_none, @@ -129,6 +129,8 @@ super( PulsarJobRunner, self ).__init__( app, nworkers, runner_param_specs=PULSAR_PARAM_SPECS, **kwds ) self._init_worker_threads() galaxy_url = self.runner_params.galaxy_url + if not galaxy_url: + galaxy_url = app.config.galaxy_infrastructure_url if galaxy_url: galaxy_url = galaxy_url.rstrip("/") self.galaxy_url = galaxy_url diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/util/lazy_process.py --- /dev/null +++ b/lib/galaxy/util/lazy_process.py @@ -0,0 +1,57 @@ +import threading +import time +import subprocess + + +class LazyProcess( object ): + """ Abstraction describing a command line launching a service - probably + as needed as functionality is accessed in Galaxy. + """ + + def __init__( self, command_and_args ): + self.command_and_args = command_and_args + self.thread_lock = threading.Lock() + self.allow_process_request = True + self.process = None + + def start_process( self ): + with self.thread_lock: + if self.allow_process_request: + self.allow_process_request = False + t = threading.Thread(target=self.__start) + t.daemon = True + t.start() + + def __start(self): + with self.thread_lock: + self.process = subprocess.Popen( self.command_and_args, close_fds=True ) + + def shutdown( self ): + with self.thread_lock: + self.allow_process_request = False + if self.running: + self.process.terminate() + time.sleep(.01) + if self.running: + self.process.kill() + + @property + def running( self ): + return self.process and not self.process.poll() + + +class NoOpLazyProcess( object ): + """ LazyProcess abstraction meant to describe potentially optional + services, in those cases where one is not configured or valid, this + class can be used in place of LazyProcess. + """ + + def start_process( self ): + return + + def shutdown( self ): + return + + @property + def running( self ): + return False diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/util/sockets.py --- /dev/null +++ b/lib/galaxy/util/sockets.py @@ -0,0 +1,43 @@ +import shlex +import socket +import subprocess +import random + + +def unused_port(range=None): + if range: + return __unused_port_on_range(range) + else: + return __unused_port_rangeless() + + +def __unused_port_rangeless(): + # TODO: Allow ranges (though then need to guess and check)... + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(('localhost', 0)) + addr, port = s.getsockname() + s.close() + return port + + +def __unused_port_on_range(range): + assert range[0] and range[1] + + # Find all ports that are already occupied + cmd_netstat = shlex.split("netstat tuln") + p1 = subprocess.Popen(cmd_netstat, stdout=subprocess.PIPE) + + occupied_ports = set() + for line in p1.stdout.read().split('\n'): + if line.startswith('tcp') or line.startswith('tcp6'): + col = line.split() + local_address = col[3] + local_port = local_address.split(':')[1] + occupied_ports.add( int(local_port) ) + + # Generate random free port number. + while True: + port = random.randrange(range[0], range[1]) + if port not in occupied_ports: + break + return port diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/visualization/registry.py --- a/lib/galaxy/visualization/registry.py +++ b/lib/galaxy/visualization/registry.py @@ -344,6 +344,10 @@ # TODO: As part of a future note plugin registry effort - move this # override and let VisualizationsRegistry just revert to using parent's # fill_template. + + # No longer needed but being left around for a few releases as ipython-galaxy + # as an external visualization plugin is deprecated in favor of core interactive + # environment plugin. if 'get_api_key' not in kwargs: def get_api_key(): @@ -401,6 +405,10 @@ """ returned = {} + # main tag specifies plugin type (visualization or + # interactive_enviornment). + returned[ 'plugin_type' ] = xml_tree.tag + # a text display name for end user links returned[ 'name' ] = xml_tree.attrib.get( 'name', None ) if not returned[ 'name' ]: diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/base/interactive_environments.py --- /dev/null +++ b/lib/galaxy/web/base/interactive_environments.py @@ -0,0 +1,162 @@ +import ConfigParser + +import hashlib +import os +import random + +from galaxy.util.bunch import Bunch +from galaxy import web +import yaml +from galaxy.managers import api_keys + + +class InteractiveEnviornmentRequest(object): + + def __init__(self, trans, plugin): + plugin_config = plugin.config + + self.trans = trans + + self.attr = Bunch() + self.attr.viz_id = plugin_config["name"].lower() + self.attr.history_id = trans.security.encode_id( trans.history.id ) + self.attr.proxy_request = trans.app.proxy_manager.setup_proxy( trans ) + self.attr.proxy_url = self.attr.proxy_request[ 'proxy_url' ] + self.attr.galaxy_config = trans.app.config + self.attr.galaxy_root_dir = os.path.abspath(self.attr.galaxy_config.root) + + self.attr.root = web.url_for("/") + self.attr.app_root = self.attr.root + "plugins/visualizations/" + self.attr.viz_id + "/static/" + + plugin_path = os.path.abspath( plugin.path ) + + # Store our template and configuration path + self.attr.our_config_dir = os.path.join(plugin_path, "config") + self.attr.our_template_dir = os.path.join(plugin_path, "templates") + self.attr.HOST = trans.request.host.rsplit(':', 1)[0] + self.attr.PORT = self.attr.proxy_request[ 'proxied_port' ] + + def load_deploy_config(self, default_dict={}): + viz_config = ConfigParser.SafeConfigParser(default_dict) + conf_path = os.path.join( self.attr.our_config_dir, self.attr.viz_id + ".ini" ) + if not os.path.exists( conf_path ): + conf_path = "%s.sample" % conf_path + viz_config.read( conf_path ) + self.attr.viz_config = viz_config + + def _boolean_option(option, default=False): + if self.attr.viz_config.has_option("main", option): + return self.attr.viz_config.getboolean("main", option) + else: + return default + + # Older style port range proxying - not sure we want to keep these around or should + # we always assume use of Galaxy dynamic proxy? None of these need to be specified + # if using the Galaxy dynamic proxy. + self.attr.PASSWORD_AUTH = _boolean_option("password_auth") + self.attr.APACHE_URLS = _boolean_option("apache_urls") + self.attr.SSL_URLS = _boolean_option("ssl") + + def write_conf_file(self, output_directory, extra={}): + """ + Build up a configuration file that is standard for ALL IEs. + + TODO: replace hashed password with plaintext. + """ + trans = self.trans + request = trans.request + api_key = api_keys.ApiKeyManager( trans.app ).get_or_create_api_key( trans.user ) + conf_file = { + 'history_id': self.attr.history_id, + 'api_key': api_key, + 'remote_host': request.remote_addr, + 'docker_port': self.attr.PORT, + 'cors_origin': request.host_url, + } + + if self.attr.viz_config.has_option("docker", "galaxy_url"): + conf_file['galaxy_url'] = self.attr.viz_config.getstring("docker", "galaxy_url") + elif self.attr.galaxy_config.galaxy_infrastructure_url_set: + conf_file['galaxy_url'] = self.attr.galaxy_config.galaxy_infrastructure_url.rstrip('/') + '/' + else: + conf_file['galaxy_url'] = request.application_url.rstrip('/') + '/' + conf_file['galaxy_paster_port'] = self.attr.galaxy_config.guess_galaxy_port() + + if self.attr.PASSWORD_AUTH: + # Generate a random password + salt + notebook_pw_salt = self.generate_password(length=12) + notebook_pw = self.generate_password(length=24) + m = hashlib.sha1() + m.update( notebook_pw + notebook_pw_salt ) + conf_file['notebook_password'] = 'sha1:%s:%s' % (notebook_pw_salt, m.hexdigest()) + # Should we use password based connection or "default" connection style in galaxy + else: + notebook_pw = "None" + + # Some will need to pass extra data + for extra_key in extra: + conf_file[extra_key] = extra[extra_key] + + self.attr.notebook_pw = notebook_pw + # Write conf + with open( os.path.join( output_directory, 'conf.yaml' ), 'wb' ) as handle: + handle.write( yaml.dump(conf_file, default_flow_style=False) ) + + def generate_hex(self, length): + return ''.join(random.choice('0123456789abcdef') for _ in range(length)) + + def generate_password(self, length): + """ + Generate a random alphanumeric password + """ + return ''.join(random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(length)) + + def javascript_boolean(self, python_boolean): + """ + Convenience function to convert boolean for use in JS + """ + if python_boolean: + return "true" + else: + return "false" + + def url_template(self, url_template): + """ + Process a URL template + + There are several variables accessible to the user: + + - ${PROXY_URL} will be replaced with dynamically create proxy + - ${PORT} will be replaced with the port the docker image is attached to + """ + # Figure out our substitutions + + # Next several lines for older style replacements (not used with Galaxy dynamic + # proxy) + + if self.attr.SSL_URLS: + protocol = 'https' + else: + protocol = 'http' + + if not self.attr.APACHE_URLS: + # If they are not using apache URLs, that implies there's a port attached to the host + # string, thus we replace just the first instance of host that we see. + url_template = url_template.replace('${HOST}', '${HOST}:${PORT}', 1) + + url_template = url_template.replace('${PROTO}', protocol) \ + .replace('${HOST}', self.attr.HOST) + + # Only the following replacements are used with Galaxy dynamic proxy + # URLs + url = url_template.replace('${PROXY_URL}', str(self.attr.proxy_url)) \ + .replace('${PORT}', str(self.attr.PORT)) + return url + + def docker_cmd(self, temp_dir): + """ + Generate and return the docker command to execute + """ + return '%s run -d --sig-proxy=true -p %s:%s -v "%s:/import/" %s' % \ + (self.attr.viz_config.get("docker", "command"), self.attr.PORT, self.attr.docker_port, + temp_dir, self.attr.viz_config.get("docker", "image")) diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/base/pluginframework.py --- a/lib/galaxy/web/base/pluginframework.py +++ b/lib/galaxy/web/base/pluginframework.py @@ -20,6 +20,8 @@ from galaxy import util from galaxy.util import odict from galaxy.util import bunch +from .interactive_environments import InteractiveEnviornmentRequest + import logging log = logging.getLogger( __name__ ) @@ -590,6 +592,11 @@ if 'plugin_path' not in kwargs: kwargs[ 'plugin_path'] = os.path.abspath( plugin.path ) + plugin_type = plugin.config["plugin_type"] + if plugin_type == "interactive_environment": + request = InteractiveEnviornmentRequest(trans, plugin) + kwargs["ie_request"] = request + # defined here to be overridden return trans.fill_template( template_filename, template_lookup=plugin.template_lookup, **kwargs ) diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/__init__.py --- /dev/null +++ b/lib/galaxy/web/proxy/__init__.py @@ -0,0 +1,169 @@ +import logging +import os +import json + +from .filelock import FileLock +from galaxy.util import sockets +from galaxy.util.lazy_process import LazyProcess, NoOpLazyProcess +from galaxy.util import sqlite + +log = logging.getLogger( __name__ ) + + +DEFAULT_PROXY_TO_HOST = "localhost" +SECURE_COOKIE = "galaxysession" + + +class ProxyManager(object): + + def __init__( self, config ): + for option in [ "manage_dynamic_proxy", "dynamic_proxy_bind_port", "dynamic_proxy_bind_ip", "dynamic_proxy_debug" ]: + setattr( self, option, getattr( config, option ) ) + self.launch_by = "node" # TODO: Support docker + if self.manage_dynamic_proxy: + self.lazy_process = self.__setup_lazy_process( config ) + else: + self.lazy_process = NoOpLazyProcess() + self.proxy_ipc = proxy_ipc(config) + + def shutdown( self ): + self.lazy_process.shutdown() + + def setup_proxy( self, trans, host=DEFAULT_PROXY_TO_HOST, port=None ): + if self.manage_dynamic_proxy: + log.info("Attempting to start dynamic proxy process") + self.lazy_process.start_process() + + authentication = AuthenticationToken(trans) + proxy_requests = ProxyRequests(host=host, port=port) + self.proxy_ipc.handle_requests(authentication, proxy_requests) + # TODO: These shouldn't need to be request.host and request.scheme - + # though they are reasonable defaults. + host = trans.request.host + if ':' in host: + host = host[0:host.index(':')] + scheme = trans.request.scheme + proxy_url = '%s://%s:%d' % (scheme, host, self.dynamic_proxy_bind_port) + return { + 'proxy_url': proxy_url, + 'proxied_port': proxy_requests.port, + 'proxied_host': proxy_requests.host, + } + + def __setup_lazy_process( self, config ): + launcher = proxy_launcher(self) + command = launcher.launch_proxy_command(config) + return LazyProcess(command) + + +def proxy_launcher(config): + return NodeProxyLauncher() + + +class ProxyLauncher(object): + + def launch_proxy_command(self, config): + raise NotImplementedError() + + +class NodeProxyLauncher(object): + + def launch_proxy_command(self, config): + args = [ + "--sessions", config.proxy_session_map, + "--ip", config.dynamic_proxy_bind_ip, + "--port", str(config.dynamic_proxy_bind_port), + ] + if config.dynamic_proxy_debug: + args.append("--verbose") + + parent_directory = os.path.dirname( __file__ ) + path_to_application = os.path.join( parent_directory, "js", "lib", "main.js" ) + command = [ path_to_application ] + args + return command + + +class AuthenticationToken(object): + + def __init__(self, trans): + self.cookie_name = SECURE_COOKIE + self.cookie_value = trans.get_cookie( self.cookie_name ) + + +class ProxyRequests(object): + + def __init__(self, host=None, port=None): + if host is None: + host = DEFAULT_PROXY_TO_HOST + if port is None: + port = sockets.unused_port() + log.info("Obtained unused port %d" % port) + self.host = host + self.port = port + + +def proxy_ipc(config): + proxy_session_map = config.proxy_session_map + if proxy_session_map.endswith(".sqlite"): + return SqliteProxyIpc(proxy_session_map) + else: + return JsonFileProxyIpc(proxy_session_map) + + +class ProxyIpc(object): + + def handle_requests(self, cookie, host, port): + raise NotImplementedError() + + +class JsonFileProxyIpc(object): + + def __init__(self, proxy_session_map): + self.proxy_session_map = proxy_session_map + + def handle_requests(self, authentication, proxy_requests): + key = "%s:%s" % ( proxy_requests.host, proxy_requests.port ) + secure_id = authentication.cookie_value + with FileLock( self.proxy_session_map ): + if not os.path.exists( self.proxy_session_map ): + open( self.proxy_session_map, "w" ).write( "{}" ) + json_data = open( self.proxy_session_map, "r" ).read() + session_map = json.loads( json_data ) + to_remove = [] + for k, value in session_map.items(): + if value == secure_id: + to_remove.append( k ) + for k in to_remove: + del session_map[ k ] + session_map[ key ] = secure_id + new_json_data = json.dumps( session_map ) + open( self.proxy_session_map, "w" ).write( new_json_data ) + + +class SqliteProxyIpc(object): + + def __init__(self, proxy_session_map): + self.proxy_session_map = proxy_session_map + + def handle_requests(self, authentication, proxy_requests): + key = "%s:%s" % ( proxy_requests.host, proxy_requests.port ) + secure_id = authentication.cookie_value + with FileLock( self.proxy_session_map ): + conn = sqlite.connect(self.proxy_session_map) + try: + c = conn.cursor() + try: + # Create table + c.execute('''CREATE TABLE gxproxy + (key text PRIMARY_KEY, secret text)''') + except Exception: + pass + insert_tmpl = '''INSERT INTO gxproxy (key, secret) VALUES ('%s', '%s');''' + insert = insert_tmpl % (key, secure_id) + c.execute(insert) + conn.commit() + finally: + conn.close() + +# TODO: RESTful API driven proxy? +# TODO: MQ diven proxy? diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/filelock.py --- /dev/null +++ b/lib/galaxy/web/proxy/filelock.py @@ -0,0 +1,82 @@ +""" Code obtained from https://github.com/dmfrey/FileLock + +See full license at: + +https://github.com/dmfrey/FileLock/blob/master/LICENSE.txt + +""" +import os +import time +import errno + + +class FileLockException(Exception): + pass + + +class FileLock(object): + """ A file locking mechanism that has context-manager support so + you can use it in a with statement. This should be relatively cross + compatible as it doesn't rely on msvcrt or fcntl for the locking. + """ + + def __init__(self, file_name, timeout=10, delay=.05): + """ Prepare the file locker. Specify the file to lock and optionally + the maximum timeout and the delay between each attempt to lock. + """ + self.is_locked = False + full_path = os.path.abspath(file_name) + self.lockfile = "%s.lock" % full_path + self.file_name = full_path + self.timeout = timeout + self.delay = delay + + def acquire(self): + """ Acquire the lock, if possible. If the lock is in use, it check again + every `wait` seconds. It does this until it either gets the lock or + exceeds `timeout` number of seconds, in which case it throws + an exception. + """ + start_time = time.time() + while True: + try: + self.fd = os.open(self.lockfile, os.O_CREAT | os.O_EXCL | os.O_RDWR) + break + except OSError as e: + if e.errno != errno.EEXIST: + raise + if (time.time() - start_time) >= self.timeout: + raise FileLockException("Timeout occured.") + time.sleep(self.delay) + self.is_locked = True + + def release(self): + """ Get rid of the lock by deleting the lockfile. + When working in a `with` statement, this gets automatically + called at the end. + """ + if self.is_locked: + os.close(self.fd) + os.unlink(self.lockfile) + self.is_locked = False + + def __enter__(self): + """ Activated when used in the with statement. + Should automatically acquire a lock to be used in the with block. + """ + if not self.is_locked: + self.acquire() + return self + + def __exit__(self, type, value, traceback): + """ Activated at the end of the with statement. + It automatically releases the lock if it isn't locked. + """ + if self.is_locked: + self.release() + + def __del__(self): + """ Make sure that the FileLock instance doesn't leave a lockfile + lying around. + """ + self.release() diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/Dockerfile --- /dev/null +++ b/lib/galaxy/web/proxy/js/Dockerfile @@ -0,0 +1,18 @@ +# Have not yet gotten this to work - goal was to launch the prox in a Docker container. +# Networking is a bit tricky though - could not get the child proxy to talk to the child +# IPython container. + + +# sudo docker build --no-cache=true -t gxproxy . +# sudo docker run --net host -v /home/john/workspace/galaxy-central/database:/var/gxproxy -p 8800:8800 -t gxproxy lib/main.js --sessions /var/gxproxy/session_map.json --ip 0.0.0.0 --port 8800 + +FROM node:0.11.13 + +RUN mkdir -p /usr/src/gxproxy +WORKDIR /usr/src/gxproxy + +ADD package.json /usr/src/gxproxy/ +RUN npm install +ADD . /usr/src/gxproxy + +CMD [ "lib/main.js" ] diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/README.md --- /dev/null +++ b/lib/galaxy/web/proxy/js/README.md @@ -0,0 +1,2 @@ +# A dynamic configurable reverse proxy for use within Galaxy + diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/lib/main.js --- /dev/null +++ b/lib/galaxy/web/proxy/js/lib/main.js @@ -0,0 +1,41 @@ +#!/usr/bin/env node +/* +Inspiration taken from + https://github.com/jupyter/multiuser-server/blob/master/multiuser/js/main.js +*/ +var fs = require('fs'); +var args = require('commander'); + +package_info = require('../package') + +args + .version(package_info) + .option('--ip <n>', 'Public-facing IP of the proxy', 'localhost') + .option('--port <n>', 'Public-facing port of the proxy', parseInt) + .option('--cookie <cookiename>', 'Cookie proving authentication', 'galaxysession') + .option('--sessions <file>', 'Routes file to monitor') + .option('--verbose') + +args.parse(process.argv); + +var DynamicProxy = require('./proxy.js').DynamicProxy; +var mapFor = require('./mapper.js').mapFor; + +var sessions = mapFor(args.sessions); + +var dynamic_proxy_options = { + sessionCookie: args['cookie'], + sessionMap: sessions, + verbose: args.verbose +} + +var dynamic_proxy = new DynamicProxy(dynamic_proxy_options); + +var listen = {}; +listen.port = args.port || 8000; +listen.ip = args.ip; + +if(args.verbose) { + console.log("Listening on " + listen.ip + ":" + listen.port); +} +dynamic_proxy.proxy_server.listen(listen.port, listen.ip); diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/lib/mapper.js --- /dev/null +++ b/lib/galaxy/web/proxy/js/lib/mapper.js @@ -0,0 +1,78 @@ +var fs = require('fs'); +var sqlite3 = require('sqlite3') + + +var endsWith = function(subjectString, searchString) { + var position = subjectString.length; + position -= searchString.length; + var lastIndex = subjectString.indexOf(searchString, position); + return lastIndex !== -1 && lastIndex === position; +}; + + +var updateFromJson = function(path, map) { + var content = fs.readFileSync(path, 'utf8'); + var keyToSession = JSON.parse(content); + var newSessions = {}; + for(var key in keyToSession) { + var hostAndPort = key.split(":"); + // 'host': hostAndPort[0], + newSessions[keyToSession[key]] = {'target': {'host': hostAndPort[0], 'port': parseInt(hostAndPort[1])}}; + } + for(var oldSession in map) { + if(!(oldSession in newSessions)) { + delete map[ oldSession ]; + } + } + for(var newSession in newSessions) { + map[newSession] = newSessions[newSession]; + } +} + +var updateFromSqlite = function(path, map) { + var newSessions = {}; + var loadSessions = function() { + db.each("SELECT key, secret FROM gxproxy", function(err, row) { + var key = row['key']; + var secret = row['secret']; + var hostAndPort = key.split(":"); + var target = {'host': hostAndPort[0], 'port': parseInt(hostAndPort[1])}; + newSessions[secret] = {'target': target}; + }, finish); + }; + + var finish = function() { + for(var oldSession in map) { + if(!(oldSession in newSessions)) { + delete map[ oldSession ]; + } + } + for(var newSession in newSessions) { + map[newSession] = newSessions[newSession]; + } + db.close(); + }; + + var db = new sqlite3.Database(path, loadSessions); +}; + + +var mapFor = function(path) { + var map = {}; + var loadMap; + if(endsWith(path, '.sqlite')) { + loadMap = function() { + updateFromSqlite(path, map); + } + } else { + loadMap = function() { + updateFromJson(path, map); + } + } + console.log("Watching path " + path); + loadMap(); + fs.watch(path, loadMap); + return map; +}; + +exports.mapFor = mapFor; \ No newline at end of file diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/lib/proxy.js --- /dev/null +++ b/lib/galaxy/web/proxy/js/lib/proxy.js @@ -0,0 +1,119 @@ +var http = require('http'), + httpProxy = require('http-proxy'); + +var bound = function (that, method) { + // bind a method, to ensure `this=that` when it is called + // because prototype languages are bad + return function () { + method.apply(that, arguments); + }; +}; + +var DynamicProxy = function(options) { + var dynamicProxy = this; + this.sessionCookie = options.sessionCookie; + this.sessionMap = options.sessionMap; + this.debug = options.verbose; + + var log_errors = function(handler) { + return function (req, res) { + try { + return handler.apply(dynamicProxy, arguments); + } catch (e) { + console.log("Error in handler for " + req.method + ' ' + req.url + ': ', e); + } + }; + }; + + var proxy = this.proxy = httpProxy.createProxyServer({ + ws : true, + }); + + this.proxy_server = http.createServer( + log_errors(dynamicProxy.handleProxyRequest) + ); + this.proxy_server.on('upgrade', bound(this, this.handleWs)); +}; + +DynamicProxy.prototype.rewriteRequest = function(request) { +} + +DynamicProxy.prototype.targetForRequest = function(request) { + // return proxy target for a given url + var session = this.findSession(request); + for (var mappedSession in this.sessionMap) { + if(session == mappedSession) { + return this.sessionMap[session].target; + } + } + + return null; +}; + +DynamicProxy.prototype.findSession = function(request) { + var sessionCookie = this.sessionCookie; + rc = request.headers.cookie; + if(!rc) { + return null; + } + var cookies = rc.split(';'); + for(var cookieIndex in cookies) { + var cookie = cookies[cookieIndex]; + var parts = cookie.split('='); + var partName = parts.shift().trim(); + if(partName == sessionCookie) { + return unescape(parts.join('=')) + } + } + + return null; +}; + +DynamicProxy.prototype.handleProxyRequest = function(req, res) { + var target = this.targetForRequest(req); + if(this.debug) { + console.log("PROXY " + req.method + " " + req.url + " to " + target); + } + var origin = req.headers.origin; + this.rewriteRequest(req); + res.oldWriteHead = res.writeHead; + res.writeHead = function(statusCode, headers) { + res.setHeader('Access-Control-Allow-Origin', origin); + res.setHeader('Access-Control-Allow-Credentials', 'true'); + res.oldWriteHead(statusCode, headers); + } + this.proxy.web(req, res, { + target: target + }, function (e) { + console.log("Proxy error: ", e); + res.writeHead(502); + res.write("Proxy target missing"); + res.end(); + }); +}; + +DynamicProxy.prototype.handleWs = function(req, res, head) { + // no local route found, time to proxy + var target = this.targetForRequest(req); + if(this.debug) { + console.log("PROXY WS " + req.url + " to " + req.url); + } + var origin = req.headers.origin; + this.rewriteRequest(req); + res.oldWriteHead = res.writeHead; + res.writeHead = function(statusCode, headers) { + res.setHeader('Access-Control-Allow-Origin', origin); + res.setHeader('Access-Control-Allow-Credentials', 'true'); + res.oldWriteHead(statusCode, headers); + } + this.proxy.ws(req, res, head, { + target: target + }, function (e) { + console.log("Proxy error: ", e); + res.writeHead(502); + res.write("Proxy target missing"); + res.end(); + }); +}; + +exports.DynamicProxy = DynamicProxy; diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 lib/galaxy/web/proxy/js/package.json --- /dev/null +++ b/lib/galaxy/web/proxy/js/package.json @@ -0,0 +1,18 @@ +{ + "name": "galaxy-proxy", + "version": "0.0.1", + "description": "A dynamic reverse proxy for use within Galaxy", + "main": "index.js", + "author": "John Chilton", + "license": "AFL v3", + "readmeFilename": "README.md", + "repository": { + "type": "mercurial", + "url": "https://bitbucket.org/galaxy/galaxy-central" + }, + "dependencies": { + "http-proxy": "~1.1", + "commander": "~2.2", + "sqlite3": "3.0.2" + } +} diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 static/scripts/galaxy.interactive_environments.js --- /dev/null +++ b/static/scripts/galaxy.interactive_environments.js @@ -0,0 +1,59 @@ +/** + * Internal function to remove content from the main area and add the notebook. + * Not idempotent + */ +function append_notebook(url){ + clear_main_area(); + $('#main').append('<iframe frameBorder="0" seamless="seamless" style="width: 100%; height: 100%; overflow:hidden;" scrolling="no" src="'+ url +'"></iframe>' + ); +} + +function clear_main_area(){ + $('#spinner').remove(); + $('#main').children().remove(); +} + +function display_spinner(){ + $('#main').append('<img id="spinner" src="' + galaxy_root + '/static/style/largespinner.gif" style="position:absolute;margin:auto;top:0;left:0;right:0;bottom:0;">'); +} + + +/** + * Test availability of a URL, and call a callback when done. + * http://stackoverflow.com/q/25390206/347368 + * @param {String} url: URL to test availability of. Must return a 200 (302->200 is OK). + * @param {String} callback: function to call once successfully connected. + * + */ +function test_ie_availability(url, success_callback){ + var request_count = 0; + display_spinner(); + interval = setInterval(function(){ + $.ajax({ + url: url, + xhrFields: { + withCredentials: true + }, + type: "GET", + timeout: 500, + success: function(){ + console.log("Connected to IE, returning"); + clearInterval(interval); + success_callback(); + }, + error: function(jqxhr, status, error){ + request_count++; + console.log("Request " + request_count); + if(request_count > 30){ + clearInterval(interval); + clear_main_area(); + toastr.error( + "Could not connect to IE, contact your administrator", + "Error", + {'closeButton': true, 'timeOut': 20000, 'tapToDismiss': false} + ); + } + } + }); + }, 1000); +} diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 static/scripts/packed/galaxy.interactive_environments.js --- /dev/null +++ b/static/scripts/packed/galaxy.interactive_environments.js @@ -0,0 +1,1 @@ +function append_notebook(a){clear_main_area();$("#main").append('<iframe frameBorder="0" seamless="seamless" style="width: 100%; height: 100%; overflow:hidden;" scrolling="no" src="'+a+'"></iframe>')}function clear_main_area(){$("#spinner").remove();$("#main").children().remove()}function display_spinner(){$("#main").append('<img id="spinner" src="'+galaxy_root+'/static/style/largespinner.gif" style="position:absolute;margin:auto;top:0;left:0;right:0;bottom:0;">')}function test_ie_availability(b,c){var a=0;display_spinner();interval=setInterval(function(){$.ajax({url:b,xhrFields:{withCredentials:true},type:"GET",timeout:500,success:function(){console.log("Connected to IE, returning");clearInterval(interval);c()},error:function(f,d,e){a++;console.log("Request "+a);if(a>30){clearInterval(interval);clear_main_area();toastr.error("Could not connect to IE, contact your administrator","Error",{closeButton:true,timeOut:20000,tapToDismiss:false})}}})},1000)}; \ No newline at end of file diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 test/unit/test_lazy_process.py --- /dev/null +++ b/test/unit/test_lazy_process.py @@ -0,0 +1,19 @@ +import os +import tempfile +import time + +from galaxy.util.lazy_process import LazyProcess + + +def test_lazy_process(): + t = tempfile.NamedTemporaryFile() + os.remove(t.name) + lazy_process = LazyProcess(["bash", "-c", "touch %s; sleep 100" % t.name]) + assert not os.path.exists(t.name) + lazy_process.start_process() + time.sleep(.02) + assert lazy_process.process.poll() is None + assert os.path.exists(t.name) + lazy_process.shutdown() + time.sleep(.02) + assert lazy_process.process.poll() diff -r 35c1392b049cbf41844be1fcf2d055f8314e16ae -r 70f7ff8c6a96ede6e439beb62d64152cc5ad2b04 test/unit/test_sockets.py --- /dev/null +++ b/test/unit/test_sockets.py @@ -0,0 +1,12 @@ +import socket +from galaxy.util import sockets + + +def test_unused_free_port_unconstrained(): + port = sockets.unused_port() + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # would throw exception if port was not free. + s.bind(('localhost', port)) + s.close() + Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.