3 new commits in galaxy-central: https://bitbucket.org/galaxy/galaxy-central/commits/d1572a4a0f52/ Changeset: d1572a4a0f52 Branch: tool_shed_checksum User: Bjoern Gruening Date: 2014-06-29 21:35:02 Summary: With that PR the Tool Shed will be able to get checksums from the Tool Developer. These checksums will be verified after download and in case of an mismatch an error is thrown. Currently, md5 and sha256 checksums are supported. Example: <package>https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5=b60639a8b2939836f66495b9a88df757</package> Affected #: 1 file diff -r 7b424e9c2e57ec12eca26f35c8f4259bbcc55cdb -r d1572a4a0f529fb20f2c7036eb7dfed19563ac89 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -8,6 +8,7 @@ import time import urllib2 import zipfile +import hashlib from galaxy.util import asbool from galaxy.util.template import fill_template @@ -139,10 +140,29 @@ class Download( object ): def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): + """ + The given download_url can have an extension like #md5# or #sha256#. + This indicates a checksum which will be chekced after download. + If the checksum does not match an exception is thrown. + + https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a... + """ + file_path = os.path.join( install_dir, downloaded_file_name ) src = None dst = None + checksum = None + sha256 = False + md5 = False # Set a timer so we don't sit here forever. + + if '#md5#' in download_url: + md5 = True + dowonload_url, checksum = download_url.split('#md5#') + elif '#sha256#' in download_url: + sha256 = True + download_url, checksum = download_url.split('#sha256#') + start_time = time.time() try: src = urllib2.urlopen( download_url ) @@ -166,6 +186,18 @@ src.close() if dst: dst.close() + + try: + if sha256: + downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() + elif md5: + downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() + + if checksum and downloaded_checksum != checksum: + raise Exception( 'Given checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) + except Exception, e: + raise + if extract: if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): archive = CompressedFile( file_path ) @@ -174,6 +206,7 @@ extraction_path = os.path.abspath( install_dir ) else: extraction_path = os.path.abspath( install_dir ) + return extraction_path @@ -1421,7 +1454,6 @@ if initial_download: filtered_actions = actions[ 1: ] env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) - log.debug( '\n%s\n' % env_shell_file_paths ) if env_shell_file_paths is None: log.debug( 'Missing Python environment, make sure your specified Python installation exists.' ) if initial_download: https://bitbucket.org/galaxy/galaxy-central/commits/848a727a1141/ Changeset: 848a727a1141 Branch: tool_shed_checksum User: BjoernGruening Date: 2014-06-29 22:34:56 Summary: spelling fix Affected #: 1 file diff -r d1572a4a0f529fb20f2c7036eb7dfed19563ac89 -r 848a727a1141aa65a0cce41f9c046ff3296bb504 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -158,7 +158,7 @@ if '#md5#' in download_url: md5 = True - dowonload_url, checksum = download_url.split('#md5#') + download_url, checksum = download_url.split('#md5#') elif '#sha256#' in download_url: sha256 = True download_url, checksum = download_url.split('#sha256#') https://bitbucket.org/galaxy/galaxy-central/commits/b86bab7dad7d/ Changeset: b86bab7dad7d User: dannon Date: 2014-07-09 17:39:45 Summary: Merged in BjoernGruening/galaxy-central-bgruening/tool_shed_checksum (pull request #432) Implement Tool Shed checksums Affected #: 1 file diff -r 4411a021a9fad918cef26e00adf6f49e247fb0bd -r b86bab7dad7d744c57f6eebe9f5b390ad083a8f6 lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py --- a/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py +++ b/lib/tool_shed/galaxy_install/tool_dependencies/recipe/step_handler.py @@ -8,6 +8,7 @@ import time import urllib2 import zipfile +import hashlib from galaxy.util import asbool from galaxy.util.template import fill_template @@ -139,10 +140,29 @@ class Download( object ): def url_download( self, install_dir, downloaded_file_name, download_url, extract=True ): + """ + The given download_url can have an extension like #md5# or #sha256#. + This indicates a checksum which will be chekced after download. + If the checksum does not match an exception is thrown. + + https://pypi.python.org/packages/source/k/khmer/khmer-1.0.tar.gz#md5#b60639a... + """ + file_path = os.path.join( install_dir, downloaded_file_name ) src = None dst = None + checksum = None + sha256 = False + md5 = False # Set a timer so we don't sit here forever. + + if '#md5#' in download_url: + md5 = True + download_url, checksum = download_url.split('#md5#') + elif '#sha256#' in download_url: + sha256 = True + download_url, checksum = download_url.split('#sha256#') + start_time = time.time() try: src = urllib2.urlopen( download_url ) @@ -166,6 +186,18 @@ src.close() if dst: dst.close() + + try: + if sha256: + downloaded_checksum = hashlib.sha256(open(file_path, 'rb').read()).hexdigest() + elif md5: + downloaded_checksum = hashlib.md5(open(file_path, 'rb').read()).hexdigest() + + if checksum and downloaded_checksum != checksum: + raise Exception( 'Given checksum does not match with the one from the downloaded file (%s).' % (downloaded_checksum) ) + except Exception, e: + raise + if extract: if tarfile.is_tarfile( file_path ) or ( zipfile.is_zipfile( file_path ) and not file_path.endswith( '.jar' ) ): archive = CompressedFile( file_path ) @@ -174,6 +206,7 @@ extraction_path = os.path.abspath( install_dir ) else: extraction_path = os.path.abspath( install_dir ) + return extraction_path @@ -1421,7 +1454,6 @@ if initial_download: filtered_actions = actions[ 1: ] env_shell_file_paths = action_dict.get( 'env_shell_file_paths', None ) - log.debug( '\n%s\n' % env_shell_file_paths ) if env_shell_file_paths is None: log.debug( 'Missing Python environment, make sure your specified Python installation exists.' ) if initial_download: Repository URL: https://bitbucket.org/galaxy/galaxy-central/ -- This is a commit notification from bitbucket.org. You are receiving this because you have the service enabled, addressing the recipient of this email.