How to know if a paramiko SSH channel is disconnected?

Question

I'm desinging test cases in which I use paramiko for SSH connections. Test cases usually contain paramiko.exec_command() calls which I have a wrapper for (called run_command()). Here self.ssh is an intance of paramiko.SSHClient(). I use a decorator to check the ssh connection before each call. (self.get_ssh() negotiates the connection)

def check_connections(function):
    ''' A decorator to check SSH connections. '''
    def deco(self, *args, **kwargs):
        if self.ssh is None:
            self.ssh = self.get_ssh()
        else:
            ret = getattr(self.ssh.get_transport(), 'is_active', None)
            if ret is None or (ret is not None and not ret()):
                self.ssh = self.get_ssh()
        return function(self, *args, **kwargs)
    return deco

@check_connections
def run_command(self, command):
    ''' Executes command via SSH. '''
    stdin, stdout, stderr = self.ssh.exec_command(command)
    stdin.flush()
    stdin.channel.shutdown_write()
    ret = stdout.read()
    err = stderr.read()
    if ret:
        return ret
    elif err:
        return err
    else:
        return None

It works perfectly until my remote node reboots, which can happen sometimes. When it occurs the next run_command() call generates a socket.error exception. The problem is, that the paramiko.Transport object seems to be remain in active state until an exception is thrown:

Python 2.7.3 (default, Mar  7 2013, 14:03:36)
[GCC 4.3.4 [gcc-4_3-branch revision 152973]] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> import paramiko
>>> ssh = paramiko.SSHClient()
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
None
>>> ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
>>> ssh.load_host_keys(os.path.expanduser('~') + '/.ssh/known_hosts')
>>> ssh.connect(hostname = '172.31.77.57', username = 'root', password = 'rootroot', timeout = 5.0)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
(<paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 1 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('reboot')
(<paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>, <paramiko.ChannelFile from <paramiko.Channel 2 (open) window=2097152 -> <paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 1 open channel(s))>>>)
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (cipher aes128-ctr, 128 bits) (active; 0 open channel(s))>
>>> print ssh.get_transport().is_active()
True
>>> ssh.exec_command('ls')
No handlers could be found for logger "paramiko.transport"
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/pytest/lib/python2.7/site-packages/paramiko/client.py", line 370, in exec_command
    chan = self._transport.open_session()
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 662, in open_session
    return self.open_channel('session')
  File "/home/pytest/lib/python2.7/site-packages/paramiko/transport.py", line 764, in open_channel
    raise e
socket.error: [Errno 104] Connection reset by peer
>>> print ssh
<paramiko.SSHClient object at 0x7f2397b96d50>
>>> print ssh.get_transport()
<paramiko.Transport at 0x97537550L (unconnected)>
>>> print ssh.get_transport().is_active()
False
>>>

Question: how can I be sure that the connection is really active or not?

Answer 1

In python, it's easier to ask for forgiveness than permission.

Wrap each call to ssh.exec_command like so:

try:
    ssh.exec_command('ls')
except socket.error as e:
    # Crap, it's closed. Perhaps reopen and retry?