Another script written with my Open Source lab developer hat on. We have a rather large number of client machines to administer, so some way of automating running commands on them is a Good Thing. We have a cron job that mounts a directory on the backend server and runs a script there, but there are occasions when this doesn't work properly or stuff needs to be done now.

Ssh is nice. So is using a private/public keypair to authenticate so you don't have to do passwords. Python is even nicer. This is what happens when you put them all together.

Update: I added the option to run a shell script remotely. It basically escapes everything in the script into a big string which it bungs on the commandline, but it works. Which brings us up to v0.2.1.

Another update: I've added threading and some error handling. You're now not limited to doing things serially. I decided that handling stdout and stderr separately was more trouble than it's worth. Synchronising the streams is non-trivial, if it's even possible, so I decided to just capture them together. Now we're at v0.3.5, and I've even put a version string in there so you can check it at runtime.

Update again: Version 0.3.6 adds locking around the output printing so that newlines don't get lost. I had been operating under the (apparently) mistaken impression the print was atomic and the newlines at the end of anything printed were occasionally going AWOL.

Changelog? It appears this is becoming something of a changelog. Ah well, that's no real problem. The major addition in v0.4.0 is the config file code. Also, locking and printing has been replaced by calls to sys.stdout.write() which is thread safe. Also, Ssh now has a single function that builds the ssh commandline for both the threaded and serial versions.

#!/usr/bin/python

import sys, os, popen2, signal

from time import sleep
from optparse import OptionParser
from ConfigParser import SafeConfigParser, NoOptionError
from threading import Thread, enumerate, currentThread

VERSION = "0.4.0"

APP_DEFAULTS = {
    "config": os.path.expanduser("~/.remote_exec"),
    "section": "default"
}

USER_DEFAULTS = {
    "section": "default",
    "host_template": "%%s",
    "machines": "localhost",
    "ssh_options": " ".join(["-o StrictHostKeyChecking=no",
                             "-o UserKnownHostsFile=hosts",
                             "-q"]),
    "user": "root",
    "keyfile": "~/.ssh/id_dsa"
}


class dummy(object):
    pid = None


class Ssh(Thread):
    options = {}

    def __init__(self, targets, commands):
        self.targets = targets
        self.commands = commands
        self.p = dummy()
        self.p.pid = None
        self.kill = False
        Thread.__init__(self)

    def build_cmd(self, target):
        target = self.options["host_template"] % (target)
        cmd = "ssh -i %s %s %s@%s %s" % (self.options["keyfile"],
                                         self.options["ssh_options"],
                                         self.options["user"],
                                         target, self.commands)
        #print "{{{%s}}}" % (cmd)
        return cmd

    def run(self):
        while not self.kill:
            try:
                target = self.targets.pop(0)
            except IndexError:
                return
            cmd = self.build_cmd(target)
            try:
                self.p = popen2.Popen4(cmd)
                for line in self.p.fromchild:
                    sys.stdout.write("[%s] %s" % (target, line))
            except IOError:
                    return

    def kill(self):
        self.kill = True
        sleep(0.01)
        if not self.p.pid is None:
            os.system("kill %s > /dev/null" % (self.p.pid))
        print "Killing thread"

    def nothreads(self):
        for target in self.targets:
            cmd = self.build_cmd(target)
            os.system(cmd)


def add_machines(option, opt, value, parser):
    """generates lists of target machines"""
    if opt == "-M" or opt == "--machinefile":
        try:
            machines_file = open(value)
        except IOError:
            parser.error("-f: file error in \"%s\"." % (value))
        machines_list = []
        for line in machines_file:
            for machine in line.split(","):
                machines_list.append(machine.strip())
        machines_file.close()
    elif opt == "-m" or opt == "--machines":
        machines_list = value.split(",")
    if not parser.values.machines: parser.values.machines = []
    for machine in machines_list:
        parser.values.machines.append(machine)


def add_commands(option, opt, value, parser):
    """reads list of commands from file"""
    try:
        commands_file = open(value)
    except IOError:
        parser.error("-c: file error in \"%s\"." % (value))
    parser.largs.append("\"")
    for line in commands_file:
        line = line.replace("\\", "\\\\")
        line = line.replace("$", "\$")
        line = line.replace("`", "\`")
        line = line.replace("\"", "\\\"")
        parser.largs.append(line)
    parser.largs.append("\"")


def make_config(parser, config_filename):
    pass


def do_config(parser):
    global APP_DEFAULTS, USER_DEFAULTS

    if parser.values.configfile:
        configfile = parser.values.configfile
    else:
        configfile = APP_DEFAULTS["config"]

    write_config = False

    cfgparser = SafeConfigParser()
    cfgparser.read(configfile)
    if parser.values.section:
        section = parser.values.section
        if not cfgparser.has_section(section):
            parser.error("Section %s (option -s) does not exist in %s." %
                         (section, configfile))
    else:
        section = APP_DEFAULTS["section"]
        if not cfgparser.has_section(section):
            cfgparser.add_section(section)
            for key, value in USER_DEFAULTS.items():
                cfgparser.set(section, key, value)
            try:
                cfgfile = open(configfile, "w")
                cfgparser.write(cfgfile)
                cfgfile.close()
            except IOError:
                parser.error("Error writing to config file %s." % configfile)
        if not cfgparser.has_option(section, "section"):
            cfgparser.set(section, "section", APP_DEFAULTS["section"])
        section = cfgparser.get(section, "section")
    if not cfgparser.has_section(section):
        parser.error("Missing section \"%s\" in %s" %
                     (section, configfile))
    if parser.values.username:
        cfgparser.set(section, "user", parser.values.username)
    if parser.values.keyfile:
        cfgparser.set(section, "keyfile", parser.values.keyfile)
    for opt in ["host_template", "ssh_options", "user", "keyfile"]:
        try:
            Ssh.options[opt] = cfgparser.get(section, opt)
        except NoOptionError:
            try:
                Ssh.options[opt] = cfgparser.get(APP_DEFAULTS["section"], opt)
            except NoOptionError:
                parser.error("missing setting \"%s\" in %s." %
                             (opt, configfile))
    if parser.values.allmachines:
        try:
            parser.values.machines = []
            for machine in cfgparser.get(section, "machines").split(","):
                parser.values.machines.append(machine.strip())
        except NoOptionError:
            parser.error("missing \"machines\" in section [%s] in %s and "
                         "--allmachines specified." % (section, configfile))


def sigint_handler(signum, frame):
    thread = currentThread()
    if thread is Ssh:
        thread.kill()
    elif thread.getName() == "MainThread":
        print "Received ctrl-c"
        sleep(0.1)
        sys.exit(1)


def main(argv=None):
    # Do command line argument parsing
    global VERSION
    if argv is None: argv = sys.argv
    parser = OptionParser(usage="usage: %prog [options] [commands to execute]",
                          version=VERSION)
    parser.set_defaults(threads=0, allmachines=False)
    parser.add_option("-m", "--machines",
                      action="callback",
                      type="string",
                      help="comma-delimited list of target machines LIST",
                      metavar="LIST",
                      dest="machines",
                      callback=add_machines)
    parser.add_option("-M", "--machinefile",
                      action="callback",
                      type="string",
                      help="list of target machines in FILE",
                      metavar="FILE",
                      dest="machines",
                      callback=add_machines)
    parser.add_option("-a", "--allmachines",
                      action="store_true",
                      help="target allmachines machines",
                      dest="allmachines")
    parser.add_option("-c", "--configfile",
                      action="store",
                      type="string",
                      help="read configuration from FILE",
                      metavar="FILE",
                      dest="configfile")
    parser.add_option("-s", "--section",
                      action="store",
                      type="string",
                      help="use settings in section SEC from config file",
                      metavar="SEC",
                      dest="section")
    parser.add_option("-u", "--user",
                      action="store",
                      type="string",
                      help="log in as USER on remote systems",
                      metavar="USER",
                      dest="username")
    parser.add_option("-k", "--key",
                      action="store",
                      type="string",
                      help="use the private key in FILE to log in",
                      metavar="FILE",
                      dest="keyfile")
    parser.add_option("-f", "--commandfile",
                      action="callback",
                      type="string",
                      help="execute commands in FILE on remote machines",
                      metavar="FILE",
                      callback=add_commands)
    parser.add_option("-t", "--threads",
                      action="store",
                      type="int",
                      help="run NUM simlutaneous threads",
                      metavar="NUM",
                      dest="threads")

    (options, args) = parser.parse_args()

    # Set various parameters
    if not (options.machines or options.allmachines):
        parser.error("You need to give target machines using -m, -f, or -a.")

    do_config(parser)

    signal.signal(signal.SIGINT, sigint_handler)

    # Do the ssh stuff we're actually here for
    if options.threads > 0:
        if len(args) == 0:
            parser.error("You cannot run interactively using -t.")
        for item in range(options.threads):
            Ssh(options.machines, " ".join(args)).start()
    else:
        Ssh(options.machines, " ".join(args)).nothreads()


if __name__ == "__main__":
    sys.exit(main())

I can't think of any more major features, so most remaining development will probably be refinement and debugging. I'm always open to new ideas, though.