details: http://www.bx.psu.edu/hg/galaxy/rev/dabbf7210eb7 changeset: 3079:dabbf7210eb7 user: Enis Afgan <afgane@gmail.com> date: Mon Nov 02 14:23:37 2009 -0500 description: Added code to process zombie instances. diffstat: lib/galaxy/cloud/providers/ec2.py | 160 +++++++++++++++++++++++++------ lib/galaxy/cloud/providers/eucalyptus.py | 160 +++++++++++++++++++++++-------- lib/galaxy/web/controllers/cloud.py | 14 ++- templates/cloud/configure_cloud.mako | 44 +++----- 4 files changed, 273 insertions(+), 105 deletions(-) diffs (581 lines): diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/cloud/providers/ec2.py --- a/lib/galaxy/cloud/providers/ec2.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/cloud/providers/ec2.py Mon Nov 02 14:23:37 2009 -0500 @@ -14,6 +14,7 @@ from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo import boto.exception +import boto import logging log = logging.getLogger( __name__ ) @@ -38,7 +39,8 @@ TERMINATED = "terminated", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -291,7 +293,8 @@ uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) # Record newly available instance data into local Galaxy database l_time = datetime.utcnow() - uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) +# uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) if not uci_wrapper.uci_launch_time_set(): uci_wrapper.set_uci_launch_time( l_time ) try: @@ -384,6 +387,7 @@ Reason behind this method is to sync state of local DB and real-world resources """ log.debug( "Running general status update for EC2 UCIs..." ) + # Update instances instances = model.CloudInstance.filter( or_( model.CloudInstance.c.state==instance_states.RUNNING, model.CloudInstance.c.state==instance_states.PENDING, model.CloudInstance.c.state==instance_states.SHUTTING_DOWN ) ).all() @@ -392,6 +396,7 @@ log.debug( "[%s] Running general status update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.instance_id ) ) self.updateInstance( inst ) + # Update storage volume(s) stores = model.CloudStore.filter( or_( model.CloudStore.c.status==store_states.IN_USE, model.CloudStore.c.status==store_states.CREATING, model.CloudStore.c.status==None ) ).all() @@ -399,6 +404,21 @@ if self.type == store.uci.credentials.provider.type: log.debug( "[%s] Running general status update on store '%s'" % ( store.uci.credentials.provider.type, store.volume_id ) ) self.updateStore( store ) + + # Attempt at updating any zombie UCIs (i.e., instances that have been in SUBMITTED state for longer than expected - see below for exact time) + zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() + for zombie in zombies: + z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ + .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, + model.CloudInstance.c.state == None ) ) \ + .all() + for z_inst in z_instances: + if self.type == z_inst.uci.credentials.provider.type: +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -406,22 +426,8 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None - + conn = self.get_connection_from_uci( inst.uci ) + # Get reservations handle for given instance try: rl= conn.get_all_instances( [inst.instance_id] ) @@ -479,21 +485,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - region=region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( inst.uci ) # Get reservations handle for given store try: @@ -531,6 +523,106 @@ uci.state( uci_states.ERROR ) return None + def processZombie( self, inst ): + """ + Attempt at discovering if starting an instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( inst.uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + if inst.keypair_name == None: + try: + inst.keypair_name = rl[0].instances[0].key_name + except: # something failed, so skip + pass + try: + state = rl[0].instances[0].update() + inst.state = state + inst.uci.state = state + inst.flush() + inst.uci.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + inst.flush() + if inst.uci.launch_time == None: + inst.uci.launch_time = launch_time + inst.uci.flush() + except: # something failed, so skip + pass + else: + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + region=region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + log.error( "Establishing connection with cloud failed: %s" % str(e) ) + uci.error( "Establishing connection with cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + return None + + return conn + # def updateUCI( self, uci ): # """ # Runs a global status update on all storage volumes and all instances that are @@ -576,7 +668,7 @@ # --------- Helper methods ------------ - def format_time( time ): + def format_time( self, time ): dict = {'T':' ', 'Z':''} for i, j in dict.iteritems(): time = time.replace(i, j) diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/cloud/providers/eucalyptus.py --- a/lib/galaxy/cloud/providers/eucalyptus.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/cloud/providers/eucalyptus.py Mon Nov 02 14:23:37 2009 -0500 @@ -6,6 +6,7 @@ from galaxy.model import mapping from galaxy.datatypes.data import nice_size from galaxy.util.bunch import Bunch +from galaxy.cloud import UCIwrapper from Queue import Queue from sqlalchemy import or_, and_ @@ -13,6 +14,8 @@ galaxy.eggs.require("boto") from boto.ec2.connection import EC2Connection from boto.ec2.regioninfo import RegionInfo +import boto.exception +import boto import logging log = logging.getLogger( __name__ ) @@ -37,7 +40,8 @@ TERMINATED = "terminated", RUNNING = "running", PENDING = "pending", - SHUTTING_DOWN = "shutting-down" + SHUTTING_DOWN = "shutting-down", + ERROR = "error" ) store_states = Bunch( @@ -256,12 +260,13 @@ uci_wrapper.set_error( "EC2 response error when starting: " + str(e), True ) l_time = datetime.utcnow() - uci_wrapper.set_launch_time( l_time, i_index=i_index ) # format_time( reservation.i_indexes[0].launch_time ) ) +# uci_wrapper.set_launch_time( l_time, i_index=i_index ) + uci_wrapper.set_launch_time( self.format_time( reservation.instances[0].launch_time ), i_index=i_index ) if not uci_wrapper.uci_launch_time_set(): uci_wrapper.set_uci_launch_time( l_time ) try: uci_wrapper.set_reservation_id( i_index, str( reservation ).split(":")[1] ) - # TODO: if more than a single instance will be started through single reservation, change this reference to element [0] + # TODO: if more than a single instance will be started through single reservation, change this reference from element [0] i_id = str( reservation.instances[0]).split(":")[1] uci_wrapper.set_instance_id( i_index, i_id ) s = reservation.instances[0].state @@ -371,15 +376,16 @@ zombies = model.UCI.filter_by( state=uci_states.SUBMITTED ).all() for zombie in zombies: z_instances = model.CloudInstance.filter_by( uci_id=zombie.id) \ - .filter( or_( model.CloudInstance.c.state!=instance_states.TERMINATED, + .filter( or_( model.CloudInstance.c.state != instance_states.TERMINATED, model.CloudInstance.c.state == None ) ) \ .all() for z_inst in z_instances: if self.type == z_inst.uci.credentials.provider.type: - log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) -# td = datetime.utcnow() - zombie.update_time -# if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes -# log.debug( "[%s] Running zombie repair update on instance '%s'" % ( inst.uci.credentials.provider.type, inst.id ) ) +# log.debug( "z_inst.id: '%s', state: '%s'" % ( z_inst.id, z_inst.state ) ) + td = datetime.utcnow() - z_inst.update_time + if td.seconds > 180: # if instance has been in SUBMITTED state for more than 3 minutes + log.debug( "[%s] Running zombie repair update on instance with DB id '%s'" % ( z_inst.uci.credentials.provider.type, z_inst.id ) ) + self.processZombie( z_inst ) def updateInstance( self, inst ): @@ -387,22 +393,7 @@ uci_id = inst.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=euca_region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( uci ) # Get reservations handle for given instance try: @@ -461,22 +452,7 @@ uci_id = store.uci_id uci = model.UCI.get( uci_id ) uci.refresh() - a_key = uci.credentials.access_key - s_key = uci.credentials.secret_key - # Get connection - try: - euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) - conn = EC2Connection( aws_access_key_id=a_key, - aws_secret_access_key=s_key, - is_secure=uci.credentials.provider.is_secure, - port=uci.credentials.provider.port, - region=euca_region, - path=uci.credentials.provider.path ) - except boto.exception.EC2ResponseError, e: - log.error( "Establishing connection with cloud failed: %s" % str(e) ) - uci.error( "Establishing connection with cloud failed: " + str(e) ) - uci.state( uci_states.ERROR ) - return None + conn = self.get_connection_from_uci( uci ) try: vl = conn.get_all_volumes( [store.volume_id] ) @@ -511,6 +487,108 @@ uci.error( "Updating volume status from cloud failed: " + str(e) ) uci.state( uci_states.ERROR ) return None + + def processZombie( self, inst ): + """ + Attempt at discovering if starting an instance was successful but local database was not updated + accordingly or if something else failed and instance was never started. Currently, no automatic + repairs are being attempted; instead, appropriate error messages are set. + """ + # Check if any instance-specific information was written to local DB; if 'yes', set instance and UCI's error message + # suggesting manual check. + if inst.launch_time != None or inst.reservation_id != None or inst.instance_id != None or inst.keypair_name != None: + # Try to recover state - this is best-case effort, so if something does not work immediately, not + # recovery steps are attempted. Recovery is based on hope that instance_id is available in local DB; if not, + # report as error. + # Fields attempting to be recovered are: reservation_id, keypair_name, instance status, and launch_time + if inst.instance_id != None: + conn = self.get_connection_from_uci( inst.uci ) + rl = conn.get_all_instances( [inst.instance_id] ) # reservation list + # Update local DB with relevant data from instance + if inst.reservation_id == None: + try: + inst.reservation_id = str(rl[0]).split(":")[1] + except: # something failed, so skip + pass + + if inst.keypair_name == None: + try: + inst.keypair_name = rl[0].instances[0].key_name + except: # something failed, so skip + pass + try: + state = rl[0].instances[0].update() + inst.state = state + inst.uci.state = state + inst.flush() + inst.uci.flush() + except: # something failed, so skip + pass + + if inst.launch_time == None: + try: + launch_time = self.format_time( rl[0].instances[0].launch_time ) + inst.launch_time = launch_time + inst.flush() + if inst.uci.launch_time == None: + inst.uci.launch_time = launch_time + inst.uci.flush() + except: # something failed, so skip + pass + else: + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance might have gotten started, manual check is recommended." + % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() + + else: #Instance most likely never got processed, so set error message suggesting user to try starting instance again. + inst.error = "Starting a machine instance associated with UCI '" + str(inst.uci.name) + "' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.state = instance_states.ERROR + inst.uci.error = "Starting a machine instance (DB id: '"+str(inst.id)+"') associated with this UCI seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." + inst.uci.state = uci_states.ERROR + log.error( "Starting a machine instance (DB id: '%s') associated with UCI '%s' seems to have failed. " \ + "Because it appears that cloud instance never got started, it should be safe to reset state and try " \ + "starting the instance again." % ( inst.id, inst.uci.name ) ) + inst.flush() + inst.uci.flush() +# uw = UCIwrapper( inst.uci ) +# log.debug( "Try automatically re-submitting UCI '%s'." % uw.get_name() ) + + def get_connection_from_uci( self, uci ): + """ + Establishes and returns connection to cloud provider. Information needed to do so is obtained + directly from uci database object. + """ + a_key = uci.credentials.access_key + s_key = uci.credentials.secret_key + # Get connection + try: + euca_region = RegionInfo( None, uci.credentials.provider.region_name, uci.credentials.provider.region_endpoint ) + conn = EC2Connection( aws_access_key_id=a_key, + aws_secret_access_key=s_key, + is_secure=uci.credentials.provider.is_secure, + port=uci.credentials.provider.port, + region=euca_region, + path=uci.credentials.provider.path ) + except boto.exception.EC2ResponseError, e: + log.error( "Establishing connection with cloud failed: %s" % str(e) ) + uci.error( "Establishing connection with cloud failed: " + str(e) ) + uci.state( uci_states.ERROR ) + return None + + return conn + # def updateUCI( self, uci ): # """ # Runs a global status update on all storage volumes and all instances that are @@ -556,7 +634,7 @@ # --------- Helper methods ------------ - def format_time( time ): + def format_time( self, time ): dict = {'T':' ', 'Z':''} for i, j in dict.iteritems(): time = time.replace(i, j) diff -r e19eef93584f -r dabbf7210eb7 lib/galaxy/web/controllers/cloud.py --- a/lib/galaxy/web/controllers/cloud.py Thu Oct 29 17:40:31 2009 -0400 +++ b/lib/galaxy/web/controllers/cloud.py Mon Nov 02 14:23:37 2009 -0500 @@ -151,7 +151,17 @@ stores = get_stores( trans, uci ) # Ensure instance is not already running (or related state) and store relevant data # into DB to initiate instance startup by cloud manager - if ( len(stores) is not 0 ) and ( uci.state == uci_states.AVAILABLE ): + if ( len(stores) is not 0 ) and \ + ( uci.state != uci_states.SUBMITTED ) and \ + ( uci.state != uci_states.SUBMITTED_UCI ) and \ + ( uci.state != uci_states.PENDING ) and \ + ( uci.state != uci_states.DELETING ) and \ + ( uci.state != uci_states.DELETING_UCI ) and \ + ( uci.state != uci_states.DELETED ) and \ + ( uci.state != uci_states.RUNNING ) and \ + ( uci.state != uci_states.NEW_UCI ) and \ + ( uci.state != uci_states.NEW ) and \ + ( uci.state != uci_states.ERROR ): instance = model.CloudInstance() instance.user = user instance.image = mi @@ -171,7 +181,7 @@ "instance description." ) return self.list( trans ) - trans.show_error_message( "Cannot start instance that is in state '%s'." % uci.state ) + error( "Cannot start instance that is in state '%s'." % uci.state ) return self.list( trans ) @web.expose diff -r e19eef93584f -r dabbf7210eb7 templates/cloud/configure_cloud.mako --- a/templates/cloud/configure_cloud.mako Thu Oct 29 17:40:31 2009 -0400 +++ b/templates/cloud/configure_cloud.mako Mon Nov 02 14:23:37 2009 -0500 @@ -24,34 +24,23 @@ $.getJSON( "${h.url_for( action='json_update' )}", {}, function ( data ) { for (var i in data) { var elem = '#' + data[i].id; - // Because of different list managing 'live' vs. 'available' instances, refresh entire - // page on necessary state change. + // Because of different list managing 'live' vs. 'available' instances, reload url on various state changes old_state = $(elem + "-state").text(); new_state = data[i].state; //console.log( "old_state[%d] = %s", i, old_state ); //console.log( "new_state[%d] = %s", i, new_state ); - if ( old_state=='pending' && new_state=='running' ) { - location.reload(true); + if ( ( old_state=='pending' && new_state=='running' ) || ( old_state=='shutting-down' && new_state=='available' ) || \ + ( old_state=='running' && new_state=='available' ) || ( old_state=='running' && new_state=='error' ) || \ + ( old_state=='pending' && new_state=='error' ) || ( old_state=='pending' && new_state=='available' ) || \ + ( old_state=='submitted' && new_state=='available' ) ) { + var url = "${h.url_for( controller='cloud', action='list')}"; + location.replace( url ); } else if ( ( old_state=='running' && new_state=='error' ) || ( old_state=='pending' && new_state=='error' ) || \ ( old_state=='submitted' && new_state=='error' ) || ( old_state=='submittedUCI' && new_state=='error' ) || \ ( old_state=='shutting-down' && new_state=='error' ) ) { - location.reload(true); - } - else if ( old_state=='shutting-down' && new_state=='available' ) { - location.reload(true); - } - else if ( old_state=='running' && new_state=='available' ) { - location.reload(true); - } - else if ( old_state=='running' && new_state=='error' ) { - location.reload(true); - } - else if ( old_state=='pending' && new_state=='error' ) { - location.reload(true); - } - else if ( old_state=='pending' && new_state=='available' ) { - location.reload(true); + var url = "${h.url_for( controller='cloud', action='list')}"; + location.replace( url ); } else if ( new_state=='shutting-down' || new_state=='shutting-downUCI' ) { $(elem + "-link").text( "" ); @@ -88,7 +77,6 @@ $(elem + "-launch_time").text( "N/A" ); } } - console.log(''); }); setTimeout("update_state()", 15000); } @@ -243,20 +231,20 @@ ${prevInstance.name} (${prevInstance.credentials.name}) <a id="pi-${i}-popup" class="popup-arrow" style="display: none;">▼</a> </td> - <td>${str(prevInstance.total_size)}</td> <!-- TODO: Change to show vol size once available--> + <td>${str(prevInstance.total_size)}</td> <td> <%state = str(prevInstance.state)%> %if state =='error': - <div id="short"> - <a onclick="document.getElementById('full').style.display = 'block'; - document.getElementById('short').style.display = 'none'; return 0" + <div id="${prevInstance.name}-short"> + <a onclick="document.getElementById('${prevInstance.name}-full').style.display = 'block'; + document.getElementById('${prevInstance.name}-short').style.display = 'none'; return 0" href="javascript:void(0)"> error </a> </div> - <div id="full" style="DISPLAY: none"> - <a onclick="document.getElementById('short').style.display = 'block'; - document.getElementById('full').style.display = 'none'; return 0;" + <div id="${prevInstance.name}-full" style="DISPLAY: none"> + <a onclick="document.getElementById('${prevInstance.name}-short').style.display = 'block'; + document.getElementById('${prevInstance.name}-full').style.display = 'none'; return 0;" href="javascript:void(0)"> error:</a><br /> ${str(prevInstance.error)}