waf/waflib/extras/remote.py

#!/usr/bin/env python
# encoding: utf-8
# Remote Builds tool using rsync+ssh

__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
__copyright__ = "Jérôme Carretero, 2013"

"""
Simple Remote Builds
********************

This tool is an *experimental* tool (meaning, do not even try to pollute
the waf bug tracker with bugs in here, contact me directly) providing simple
remote builds.

It uses rsync and ssh to perform the remote builds.
It is intended for performing cross-compilation on platforms where
a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
does not exist (eg. Windows builds using Visual Studio) or simply not installed.
This tool sends the sources and the waf script to the remote host,
and commands the usual waf execution.

There are alternatives to using this tool, such as setting up shared folders,
logging on to remote machines, and building on the shared folders.
Electing one method or another depends on the size of the program.


Usage
=====

1. Set your wscript file so it includes a list of variants,
   e.g.::

     from waflib import Utils
     top = '.'
     out = 'build'

     variants = [
      'linux_64_debug',
      'linux_64_release',
      'linux_32_debug',
      'linux_32_release',
      ]

     from waflib.extras import remote

     def options(opt):
         # normal stuff from here on
         opt.load('compiler_c')

     def configure(conf):
         if not conf.variant:
             return
         # normal stuff from here on
         conf.load('compiler_c')

     def build(bld):
         if not bld.variant:
             return
         # normal stuff from here on
         bld(features='c cprogram', target='app', source='main.c')


2. Build the waf file, so it includes this tool, and put it in the current
   directory

   .. code:: bash

      ./waf-light --tools=remote

3. Set the host names to access the hosts:

   .. code:: bash

      export REMOTE_QNX=user@kiunix

4. Setup the ssh server and ssh keys

   The ssh key should not be protected by a password, or it will prompt for it every time.
   Create the key on the client:

   .. code:: bash

      ssh-keygen -t rsa -f foo.rsa

   Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
   and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)

   A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
   The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
   be warned to use this feature on internal networks only (MITM).

   .. code:: bash

      export WAF_SSH_KEY=~/foo.rsa

5. Perform the build:

   .. code:: bash

      waf configure_all build_all --remote

"""


import getpass, os, re, sys
from collections import OrderedDict
from waflib import Context, Options, Utils, ConfigSet

from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
from waflib.Configure import ConfigurationContext


is_remote = False
if '--remote' in sys.argv:
	is_remote = True
	sys.argv.remove('--remote')

class init(Context.Context):
	"""
	Generates the *_all commands
	"""
	cmd = 'init'
	fun = 'init'
	def execute(self):
		for x in list(Context.g_module.variants):
			self.make_variant(x)
		lst = ['remote']
		for k in Options.commands:
			if k.endswith('_all'):
				name = k.replace('_all', '')
				for x in Context.g_module.variants:
					lst.append('%s_%s' % (name, x))
			else:
				lst.append(k)
		del Options.commands[:]
		Options.commands += lst

	def make_variant(self, x):
		for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
			name = y.__name__.replace('Context','').lower()
			class tmp(y):
				cmd = name + '_' + x
				fun = 'build'
				variant = x
		class tmp(ConfigurationContext):
			cmd = 'configure_' + x
			fun = 'configure'
			variant = x
			def __init__(self, **kw):
				ConfigurationContext.__init__(self, **kw)
				self.setenv(x)

class remote(BuildContext):
	cmd = 'remote'
	fun = 'build'

	def get_ssh_hosts(self):
		lst = []
		for v in Context.g_module.variants:
			self.env.HOST = self.login_to_host(self.variant_to_login(v))
			cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
			out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
			lst.append(out.strip())
		return lst

	def setup_private_ssh_key(self):
		"""
		When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
		Make sure that the ssh key does not prompt for a password
		"""
		key = os.environ.get('WAF_SSH_KEY', '')
		if not key:
			return
		if not os.path.isfile(key):
			self.fatal('Key in WAF_SSH_KEY must point to a valid file')
		self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
		self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
		self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
		self.ssh_config = os.path.join(self.ssh_dir, 'config')
		for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
			if not os.path.isfile(x):
				if not os.path.isdir(self.ssh_dir):
					os.makedirs(self.ssh_dir)
				Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
				os.chmod(self.ssh_key, 448)

				Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
				os.chmod(self.ssh_key, 448)

				Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
				os.chmod(self.ssh_config, 448)
		self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
		self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')

	def skip_unbuildable_variant(self):
		# skip variants that cannot be built on this OS
		for k in Options.commands:
			a, _, b = k.partition('_')
			if b in Context.g_module.variants:
				c, _, _ = b.partition('_')
				if c != Utils.unversioned_sys_platform():
					Options.commands.remove(k)

	def login_to_host(self, login):
		return re.sub(r'(\w+@)', '', login)

	def variant_to_login(self, variant):
		"""linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
		x = variant[:variant.rfind('_')]
		ret = os.environ.get('REMOTE_' + x.upper(), '')
		if not ret:
			x = x[:x.find('_')]
			ret = os.environ.get('REMOTE_' + x.upper(), '')
		if not ret:
			ret = '%s@localhost' % getpass.getuser()
		return ret

	def execute(self):
		global is_remote
		if not is_remote:
			self.skip_unbuildable_variant()
		else:
			BuildContext.execute(self)

	def restore(self):
		self.top_dir = os.path.abspath(Context.g_module.top)
		self.srcnode = self.root.find_node(self.top_dir)
		self.path = self.srcnode

		self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
		self.bldnode = self.root.make_node(self.out_dir)
		self.bldnode.mkdir()

		self.env = ConfigSet.ConfigSet()

	def extract_groups_of_builds(self):
		"""Return a dict mapping each variants to the commands to build"""
		self.vgroups = {}
		for x in reversed(Options.commands):
			_, _, variant = x.partition('_')
			if variant in Context.g_module.variants:
				try:
					dct = self.vgroups[variant]
				except KeyError:
					dct = self.vgroups[variant] = OrderedDict()
				try:
					dct[variant].append(x)
				except KeyError:
					dct[variant] = [x]
				Options.commands.remove(x)

	def custom_options(self, login):
		try:
			return Context.g_module.host_options[login]
		except (AttributeError, KeyError):
			return {}

	def recurse(self, *k, **kw):
		self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
		self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
		self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
		try:
			self.env.WAF = getattr(Context.g_module, 'waf')
		except AttributeError:
			try:
				os.stat('waf')
			except KeyError:
				self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
			else:
				self.env.WAF = './waf'

		self.extract_groups_of_builds()
		self.setup_private_ssh_key()
		for k, v in self.vgroups.items():
			task = self(rule=rsync_and_ssh, always=True)
			task.env.login = self.variant_to_login(k)

			task.env.commands = []
			for opt, value in v.items():
				task.env.commands += value
			task.env.variant = task.env.commands[0].partition('_')[2]
			for opt, value in self.custom_options(k):
				task.env[opt] = value
		self.jobs = len(self.vgroups)

	def make_mkdir_command(self, task):
		return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)

	def make_send_command(self, task):
		return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)

	def make_exec_command(self, task):
		txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
		return Utils.subst_vars(txt, task.env)

	def make_save_command(self, task):
		return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)

def rsync_and_ssh(task):

	# remove a warning
	task.uid_ = id(task)

	bld = task.generator.bld

	task.env.user, _, _ = task.env.login.partition('@')
	task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
	task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
	task.env.local_dir = bld.srcnode.abspath() + '/'

	task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
	task.env.build_dir = bld.bldnode.abspath()

	ret = task.exec_command(bld.make_mkdir_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_send_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_exec_command(task))
	if ret:
		return ret
	ret = task.exec_command(bld.make_save_command(task))
	if ret:
		return ret