Update both parts for proper modularity, start catalyst-based backend, code cleanup, especially on backend

17 files changed, 385 insertions, 166 deletions

diff --git a/backend/backend.php b/backend/backend.php
index 4e2209a..cde76f0 100755
--- a/backend/backend.php
+++ b/backend/backend.php
@@ -1,8 +1,5 @@
 #!/usr/bin/php
 <?php
-require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out (and 2 lines down)
-require_once(BACKEND.'/include/signals.php');
-declare(ticks=1);
 $pidfile='/var/run/ingenue.pid'; // Doesn't work when not run as root
 $opts=getopt('fk');
 if (isset($opts['f'])) {
@@ -21,19 +18,19 @@ if (isset($opts['k'])) {
 	if (is_file($pidfile)) {
 		$pid=trim(file_get_contents($pidfile));
 		if (posix_kill($pid, 0)) {
-			debug("Sending SIGTERM to $pid");
+			fputs(STDERR, "Sending SIGTERM to $pid");
 			if (!posix_kill($pid, SIGTERM)) {
-				debug("Failed to send SIGTERM to $pid");
+				fputs(STDERR, "Failed to send SIGTERM to $pid");
 				die(1);
 			} else {
 				// TODO wait for $pid to exit
 			}
 		} else {
-			debug("Couldn't send signal 0 to $pid");
+			fputs(STDERR, "Couldn't send signal 0 to $pid");
 			die(0);
 		}
 	} else {
-		debug('No PID file found');
+		fputs(STDERR, 'No PID file found');
 	}
 	die();
 }
@@ -43,9 +40,12 @@ if (is_file($pidfile)) {
 		die("Found already running backend PID=$pid.\n");
 }
 if (posix_geteuid() !== 0)
-	debug("Not running as root... this is not going to accomplish much.");
+	fputs(STDERR, "Not running as root... this is not going to accomplish much.");
 if (file_put_contents($pidfile, posix_getpid()))
 	$unlinkpidfile=true;
+require_once(dirname(__FILE__).'/../shared/include/includes.php'); // USE __DIR__ once 5.3.0 is out (and 2 lines down)
+require_once(BACKEND.'/include/signals.php');
+declare(ticks=1);
 require_once(SHARED.'/include/dbinit.php');
 while (true) {
 	// TODO check first for builds that need to be resumed (and figure out how to resume things)
@@ -58,7 +58,19 @@ while (true) {
 		log_msg('Starting build id='.$build->id);
 		$success=null;
 		try {
-			$success=build($build);
+			$opts=$build->get_buildopts();
+			$module=$opts['backend_module'];
+			$build_proc=$module.'_build';
+			foreach (glob(BACKEND."/modules/$module/*.php") as $inc) {
+				require_once($inc);
+			}
+			// TODO check that build_proc exists
+			$workdir=WORK.'/build-'.$build->id;
+			fatal(log_status('Creating work directory '.$workdir, mkdir($workdir, 0700)));
+			$success=$build_proc($build, $opts, $workdir);
+			if (!$conf['debug']) {
+				execute_command('Delete work directory', 'rm -rf "'.$workdir.'"');
+			}
 		} catch (Exception $e) {
 			log_msg('Caught exception: '.$e->getMessage());
 			$build->status='finished/failed: '.$e->getMessage();
@@ -75,8 +87,8 @@ while (true) {
 		$build->write();
 		unset($build);
 	}
-	log_msg('Sleeping...', false);
+//	log_msg('Sleeping...', false);
 	sleep(5);
-	log_msg("done");
+//	log_msg("done");
 }
 ?>
diff --git a/backend/functions/build.php b/backend/functions/build.php
deleted file mode 100644
index a5e2058..0000000
--- a/backend/functions/build.php
+++ /dev/null
@@ -1,90 +0,0 @@
-<?php
-// This is the main function that carries out a build from start to finish
-function build(&$build) {
-	global $conf, $profile;
-	$opts=$build->get_buildopts();
-	$profile=new sql_profile($opts['profile']);
-	$headers=$profile->get_headers();
-	$makeconf['pkgdir']=$conf['pkgdir_root'].'/'.$profile->pkgdir;
-	$makeconf['chost']=$headers['chost'];
-	$makeconf['accept_keywords']=$headers['accept_keywords'];
-	$build->write();
-	$W=WORK.'/build-'.$build->id;
-	fatal(log_status('Creating work directory '.$W, mkdir($W, 0700)));
-	$I=$W.'/image';
-	fatal(log_status('Creating '.$I, mkdir($I, 0700)));
-	$makeconf['root']=$I;
-	$C=$W.'/config_root';
-	fatal(log_status('Creating '.$C, mkdir($C, 0700)));
-	fatal(log_status("Making symlink $C/etc -> $C", symlink('.', "$C/etc")));
-	fatal(log_status('Creating '.$W.'/log', mkdir($W.'/log', 0700)));
-	$makeconf['port_logdir']=$W.'/log';
-	$makeconf['emerge_log_dir']=$conf['port_logdir'];
-	fatal(log_status('Creating '.$W.'/tmp', mkdir($W.'/tmp', 0700)));
-	$makeconf['portage_tmpdir']=$W.'/tmp';
-	$makeconf['emerge_default_opts']=$conf['emerge_default_opts'];
-	$contents='';
-	foreach ($makeconf as $name => $val) {
-		$contents.=strtoupper($name).'='.escapeshellarg($val)."\n";
-	}
-	unset($makeconf);
-	fatal(log_status('Writing '.$C.'/make.conf', file_put_contents($C.'/etc/make.conf', $contents)));
-	unset($contents);
-	fatal(log_status('Making make.profile symlink to '.$conf['portdir'].'/profiles/'.$headers['profile'], symlink($conf['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')));
-	$env=array(
-		'PORTAGE_CONFIGROOT' => $C,
-		'PATH' => $_ENV['PATH']
-	);
-	sql_task::execute_command('Log portage setup', 'emerge --info', $build, true, null, $env);
-//	sql_task::execute_command('Pre-installing baselayout', 'emerge baselayout', $build, true, null, array_merge($env, array('USE' => 'build')));
-	//	TODO create make.conf, make.profile in target /etc
-	sql_task::execute_command('Install base system', 'emerge system', $build, true, null, $env);
-	if (isset($opts['image_type']) && $opts['image_type'] == 'livecd')
-		sql_task::execute_command('Install LiveCD utilities', 'emerge livecd-tools', $build, true, null, $env);
-	if (isset($opts['install_packages'])) {
-		$pkgs=array();
-		foreach (explode(' ', $opts['install_packages']) as $atom) {
-			$pkgs[]=escapeshellarg($atom);
-		}
-		if (count($pkgs) > 1)
-			sql_task::execute_command('Check that extra packages will install', 'emerge -p '.implode(' ', $pkgs), $build, true, null, $env);
-		foreach ($pkgs as $atom)
-			sql_task::execute_command('Install extra package '.$atom, 'emerge '.$atom, $build, true, null, $env);
-	}
-	$imgtype=isset($opts['image_type'])?$opts['image_type']:'tbz2';
-	if ($imgtype == 'tbz2') {
-		sql_task::execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$W/image' .", $build, true, null, $env);
-		rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed');
-	} elseif ($imgtype == 'tgz') {
-		sql_task::execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$W/image' .", $build, true, null, $env);
-		rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed');
-	} elseif ($imgtype == 'livecd' || $imgtype == 'installcd') {
-		if (strpos($headers['chost'], 'x86_64') === false)
-			$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
-		else
-			$minimaliso='/home/eitan/soc/install-amd64-minimal-20090625.iso';
-		sql_task::execute_command('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'", $build, true, null, $env);
-		sql_task::execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'", $build, true, null, $env);
-		sql_task::execute_command('Unmount CD image', "umount '$W/tmp'", $build, true, null, $env);
-		sql_task::execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$W/image/boot/'", $build, true, null, $env);
-		file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
-		sql_task::execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$W/image' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'", $build, true, null, $env);
-		if ($imgtype == 'livecd') {
-			rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS');
-			sql_task::execute_command('Compress finished image to squashfs', "mksquashfs '$W/image' '$W/cd/image.squashfs' -noappend -info", $build, true, null, $env);
-		} else // Install CD
-			sql_task::execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$W/image' .", $build, true, null, $env);
-		// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
-		// ISOLINUX bootloader
-		// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
-		sql_task::execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'", $build, true, null, $env);
-		rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed');
-	} else {
-		throw_exception('invalid image type: '.$imgtype);
-	}
-	if (!$conf['debug']) {
-		sql_task::execute_command('Delete work directory', 'rm -rf "'.$W.'"', $build, true, null, $env);
-	}
-	return true;
-}
-?>
diff --git a/backend/functions/execution.php b/backend/functions/execution.php
new file mode 100644
index 0000000..dd6dd88
--- /dev/null
+++ b/backend/functions/execution.php
@@ -0,0 +1,40 @@
+<?php
+function execute_command_with_all($description, $command, $fatal=true, $path=null, $env=null) {
+	global $build, $task;
+	$default_env=array(
+		'PATH' => $_ENV['PATH']
+	);		
+	$env=is_array($env)?array_merge($default_env, $env):$default_env;
+	// TODO this won't work once we have internal tasks too - we need to use a common function for tracking order
+	static $buildid=null;
+	static $order=0;
+	if ($build->id !== $buildid) {
+		$buildid=$build->id;
+		$order=0;
+	}
+	$task=new sql_task($build->id, $order++, null, $description, $command);
+	$result=$task->execute($path, $env);
+	unset($task);
+	if ($result != 0 && $fatal) {
+		if ($result > 0)
+			throw_exception($command.' returned with exit status '.$result);
+		elseif ($result == -128)
+			throw_exception($command.' received an unknown signal');
+		else
+			throw_exception($command.' received signal '.-$result);
+	}
+	return $result;
+}
+function execute_command($desc, $cmd) {
+	return execute_command_with_all($desc, $cmd, true, null, null);
+}
+function execute_command_with_env($desc, $cmd, $env) {
+	return execute_command_with_all($desc, $cmd, true, null, $env);
+}
+function execute_command_with_path($desc, $cmd, $path) {
+	return execute_command_with_all($desc, $cmd, true, $path, null);
+}
+function execute_non_fatal_command($desc, $cmd, $path=null, $env=null) {
+	return execute_command_with_all($desc, $cmd, false, $path, $env);
+}
+?>
diff --git a/backend/functions/makedirs.php b/backend/functions/makedirs.php
new file mode 100644
index 0000000..eb572c9
--- /dev/null
+++ b/backend/functions/makedirs.php
@@ -0,0 +1,18 @@
+<?php
+function makedirs() {
+	for ($i=0; $i<func_num_args(); $i++) {
+		$dir=func_get_arg($i);
+		if (is_array($dir)) {
+			call_user_func('makedirs', $dir);
+		} else {
+			makedir($dir);
+		}
+	}
+}
+function makedir($dir) {
+	global $workdir;
+	if (substr($dir, 0, 1) != '/')
+		$dir="$workdir/$dir";
+	fatal(log_status('Creating '.$dir, mkdir($dir, 0700, true)));
+}
+?>
diff --git a/backend/modules/gentoo_catalyst/build.php b/backend/modules/gentoo_catalyst/build.php
new file mode 100644
index 0000000..45b2b6e
--- /dev/null
+++ b/backend/modules/gentoo_catalyst/build.php
@@ -0,0 +1,71 @@
+<?php
+function gentoo_catalyst_build(&$build, &$opts, &$W) {
+	global $conf, $profile;
+	$profile=new sql_profile($opts['profile']);
+	$headers=$profile->get_headers();
+	$arch=explode(' ', $headers['accept_keywords']);
+	$arch=$arch[0];
+	if (substr($arch, 0, 1) == '~')
+		$arch=substr($arch, 1);
+//	$C=$W.'/config_root';
+	makedirs('catalyst_tmp', 'snapshot_cache');
+	$catalyst=array(
+		'digests' => '',
+		'contents' => '',
+		'distdir' => '',
+		// 'envscript' => '',
+		'hash_function' => 'crc32',
+		'options' => 'pkgcache metadata_overlay seedcache snapcache',
+		'portdir' => $conf['portdir'],
+		'sharedir' => '/usr/lib/catalyst', // This may not work on all systems
+		'snapshotcache' => "$W/snapshot_cache",
+		'storedir' => "$W/catalyst_tmp"
+	);
+	$ctc=fopen("$W/catalyst.conf", 'w');
+	foreach ($catalyst as $name => $val) {
+		fputs($ctc, "$name=\"$val\"\n");
+	}
+	fclose($ctc);
+	// TODO - stages 1-3 first, only do stage4 if we have extra packages to install
+	$spec=array(
+		'subarch' => $arch,
+		'version_stamp' => gmdate('Ymd-His'),
+		'target' => 'stage4',
+		'rel_type' => 'default',
+		'profile' => $headers['profile'],
+		'snapshot' => gmdate('Ymd'),
+		'source_subpath' => 'default', // FIXME
+		'portage_confdir' => '',
+		'portage_overlay' => '',
+		'pkgcache_path' => "$W/pkgcache",
+		'kerncache_path' => '',
+		'stage4/use' => $headers['use'],
+		'stage4/packages' => $opts['install_packages'],
+		'stage4/fsscript' => '',
+		'stage4/splash_theme' => '',
+		'stage4/gk_mainargs' => '',
+		'stage4/linuxrc' => '',
+		'stage4/motd' => '',
+//		'stage4/modblacklist' => '8139cp', // Not recognized by catalyst
+		'stage4/rcadd' => '',
+		'stage4/rcdel' => '',
+		'stage4/root_overlay' => '',
+		'stage4/xinitrc' => '',
+		'stage4/users' => '',
+		'boot/kernel' => '',
+		'stage4/unmerge' => '',
+		'stage4/empty' => '/var/tmp /var/cache /var/db /var/empty /var/lock /var/log /var/run /var/spool /var/state /tmp /usr/portage /usr/share/man /usr/share/info /usr/share/unimaps /usr/include /usr/share/zoneinfo /usr/share/dict /usr/share/doc /usr/share/ss /usr/share/state /usr/share/texinfo /usr/lib/python2.2 /usr/lib/portage /usr/share/gettext /usr/share/i18n /usr/share/rfc /usr/lib/X11/config /usr/lib/X11/etc /usr/lib/X11/doc /usr/src /usr/share/doc /usr/share/man /root/.ccache /etc/cron.daily /etc/cron.hourly /etc/cron.monthly /etc/cron.weekly /etc/logrotate.d /etc/rsync /usr/lib/awk /usr/lib/ccache /usr/lib/gcc-config /usr/lib/nfs /usr/local /usr/diet/include /usr/diet/man /usr/share/consolefonts/partialfonts /usr/share/consoletrans /usr/share/emacs /usr/share/gcc-data /usr/share/genkernel /etc/splash/gentoo /etc/splash/emergence /usr/share/gnuconfig /usr/share/lcms /usr/share/locale /etc/skel',
+		'stage4/rm' => '/lib/*.a /usr/lib/*.a /usr/lib/gcc-lib/*/*/libgcj* /etc/dispatch-conf.conf /etc/etc-update.conf /etc/*- /etc/issue* /etc/make.conf /etc/man.conf /etc/*.old /root/.viminfo /usr/sbin/fb* /usr/sbin/fsck.cramfs /usr/sbin/fsck.minix /usr/sbin/mkfs.minix /usr/sbin/mkfs.bfs /usr/sbin/mkfs.cramfs /lib/security/pam_access.so /lib/security/pam_chroot.so /lib/security/pam_debug.so /lib/security/pam_ftp.so /lib/security/pam_issue.so /lib/security/pam_mail.so /lib/security/pam_motd.so /lib/security/pam_mkhomedir.so /lib/security/pam_postgresok.so /lib/security/pam_rhosts_auth.so /lib/security/pam_userdb.so /usr/share/consolefonts/1* /usr/share/consolefonts/7* /usr/share/consolefonts/8* /usr/share/consolefonts/9* /usr/share/consolefonts/A* /usr/share/consolefonts/C* /usr/share/consolefonts/E* /usr/share/consolefonts/G* /usr/share/consolefonts/L* /usr/share/consolefonts/M* /usr/share/consolefonts/R* /usr/share/consolefonts/a* /usr/share/consolefonts/c* /usr/share/consolefonts/dr* /usr/share/consolefonts/g* /usr/share/consolefonts/i* /usr/share/consolefonts/k* /usr/share/consolefonts/l* /usr/share/consolefonts/r* /usr/share/consolefonts/s* /usr/share/consolefonts/t* /usr/share/consolefonts/v* /etc/splash/livecd-2006.1/16* /etc/splash/livecd-2006.1/12* /etc/splash/livecd-2006.1/6* /etc/splash/livecd-2006.1/8* /etc/splash/livecd-2006.1/images/silent-16* /etc/splash/livecd-2006.1/images/silent-12* /etc/splash/livecd-2006.1/images/silent-6* /etc/splash/livecd-2006.1/images/silent-8* /etc/splash/livecd-2006.1/images/verbose-16* /etc/splash/livecd-2006.1/images/verbose-12* /etc/splash/livecd-2006.1/images/verbose-6* /etc/splash/livecd-2006.1/images/verbose-8* /etc/make.conf.example /etc/make.globals /etc/resolv.conf'
+	);
+	$specfile=fopen("$W/stage4.spec", 'w');
+	foreach ($spec as $name => $val) {
+		if (strlen($val))
+			fputs($specfile, "$name: $val\n");
+	}
+	fclose($specfile);
+	$pkgdir=$spec['pkgcache_path'].'/'.$spec['rel_type'].'/'.$spec['target'].'-'.$spec['version_stamp'];
+	makedir(dirname($pkgdir));
+	symlink($pkgdir, $profile->pkgdir);
+	execute_command('Running Catalyst for build stage 4', "catalyst -c '$W/catalyst.conf' -f '$W/stage4.spec'");
+}
+?>
diff --git a/backend/modules/gentoo_catalyst/catalyst.conf b/backend/modules/gentoo_catalyst/catalyst.conf
new file mode 100644
index 0000000..ba5505b
--- /dev/null
+++ b/backend/modules/gentoo_catalyst/catalyst.conf
@@ -0,0 +1,79 @@
+# /etc/catalyst/catalyst.conf
+
+# Simple desriptions of catalyst settings. Please refer to the online
+# documentation for more information.
+
+# Creates a .DIGESTS file containing the hash output from any of the supported
+# options below.  Adding them all may take a long time.
+# Supported options: sha1, sha224, ripemd128, ripemd320, sha384, crc32,
+#	ripemd256, sha256, sha512, ripemd160, md5
+digests="md5 sha1"
+
+# Creates a .CONTENTS file listing the contents of the file. Pick from any of
+# the supported options below:
+# auto		- strongly recommended
+# tar-tv	- does 'tar tvf FILE'
+# tar-tvz	- does 'tar tvzf FILE'
+# tar-tvy	- does 'tar tvyf FILE'
+# isoinfo-l	- does 'isoinfo -l -i FILE'
+# isoinfo-f	- does 'isoinfo -f -i FILE'
+# 'isoinfo-f' is the only option not chosen by the automatic algorithm.
+# If this variable is empty, no .CONTENTS will be generated at all.
+contents="auto"
+
+# distdir specifies where your distfiles are located. This setting should
+# work fine for most default installations.
+distdir="/usr/portage/distfiles"
+
+# envscript allows users to set options such as http proxies, MAKEOPTS,
+# GENTOO_MIRRORS, or any other environment variables needed for building.
+# The envscript file sets environment variables like so:
+# export FOO="bar"
+envscript="/etc/catalyst/catalystrc"
+
+# Internal hash function catalyst should use for things like autoresume, 
+# seedcache, etc.  The default and fastest is crc32.  You should not ever need
+# to change this unless your OS does not support it.
+# Supported options: sha1, sha224, ripemd128, ripemd320, sha384, crc32,
+#	ripemd256, sha256, sha512, ripemd160, md5
+hash_function="crc32"
+
+# options set different build-time options for catalyst. Some examples are:
+# autoresume = Attempt to resume a failed build, clear the autoresume flags with
+#	the -a option to the catalyst cmdline.  -p will clear the autoresume flags
+#	as well as your pkgcache and kerncache
+#	( This option is not fully tested, bug reports welcome )
+# ccache = enables build time ccache support (highly recommended)
+# distcc = enable distcc support for building. You have to set distcc_hosts in
+# 	your spec file.
+# icecream = enables icecream compiler cluster support for building
+# kerncache = keeps a tbz2 of your built kernel and modules (useful if your
+#	build stops in livecd-stage2)
+# metadata_overlay = enabled the metadata_overlay cache module in portage, which
+#	uses the in-tree metadata
+# pkgcache = keeps a tbz2 of every built package (useful if your build stops
+#	prematurely)
+# seedcache = use the build output of a previous target if it exists to speed up
+#	the copy
+# snapcache = cache the snapshot so that it can be bind-mounted into the chroot.
+#	WARNING: moving parts of the portage tree from within fsscript *will* break
+#	your cache. The cache is unlinked before any empty or rm processing, though.
+#
+# (These options can be used together)
+options="autoresume kerncache metadata_overlay pkgcache seedcache snapcache"
+
+# portdir specifies the source portage tree used by the snapshot target.
+portdir="/usr/portage"
+
+# sharedir specifies where all of the catalyst runtime executables are. Most
+# users do not need to change this.
+sharedir="/usr/lib64/catalyst"
+
+# snapshot_cache specifies where the snapshots will be cached to if snapcache is
+# enabled in the options.
+snapshot_cache="/var/tmp/catalyst/snapshot_cache"
+
+# storedir specifies where catalyst will store everything that it builds, and
+# also where it will put its temporary files and caches.
+storedir="/var/tmp/catalyst"
+
diff --git a/backend/modules/gentoo_catalyst/catalystrc b/backend/modules/gentoo_catalyst/catalystrc
new file mode 100644
index 0000000..581772d
--- /dev/null
+++ b/backend/modules/gentoo_catalyst/catalystrc
@@ -0,0 +1,6 @@
+#!/bin/bash
+# This is an example catalystrc. As such, it doesn't actually *do* anything.
+
+# Uncomment the following to increase the number of threads used to compile.
+# export MAKEOPTS="-j16"
+
diff --git a/backend/modules/gentoo_portage/build.php b/backend/modules/gentoo_portage/build.php
new file mode 100644
index 0000000..693a85c
--- /dev/null
+++ b/backend/modules/gentoo_portage/build.php
@@ -0,0 +1,78 @@
+<?php
+// This is the main function that carries out a build from start to finish
+function gentoo_portage_build(&$build, &$opts, &$W) {
+	global $conf;
+	$profile=new sql_profile($opts['profile']);
+	$headers=$profile->get_headers();
+	$I=$W.'/image';
+	$C=$W.'/config_root';
+	makedirs($I, $C, "$W/log", "$W/tmp");
+	fatal(log_status("Making symlink $C/etc -> .", symlink('.', "$C/etc")));
+	$makeconf=array(
+		'pkgdir' => $conf['pkgdir_root'].'/'.$profile->pkgdir,
+		'chost' => $headers['chost'],
+		'accept_keywords' => $headers['accept_keywords'],
+		'root' => $I,
+		'port_logdir' => "$W/log",
+		'emerge_log_dir' => "$W/log",
+		'portage_tmpdir' => "$W/tmp",
+		'emerge_default_opts' => $conf['emerge_default_opts']
+	);
+	$contents='';
+	foreach ($makeconf as $name => $val)
+		$contents.=strtoupper($name).'='.escapeshellarg($val)."\n";
+	unset($makeconf);
+	fatal(log_status('Writing '.$C.'/make.conf', file_put_contents($C.'/etc/make.conf', $contents)));
+	unset($contents);
+	fatal(log_status('Making make.profile symlink to '.$conf['portdir'].'/profiles/'.$headers['profile'], symlink($conf['portdir'].'/profiles/'.$headers['profile'], $C.'/etc/make.profile')));
+	$prtg_cfgrt=array('PORTAGE_CONFIGROOT' => $C);
+	execute_command_with_env('Log portage setup', 'emerge --info', $prtg_cfgrt);
+//	execute_command_with_env('Pre-installing baselayout', 'emerge baselayout', array_merge($prtg_cfgrt, array('USE' => 'build')));
+	//	TODO create make.conf, make.profile in target /etc
+	execute_command_with_env('Install base system', 'emerge system', $prtg_cfgrt);
+	if (isset($opts['image_type']) && $opts['image_type'] == 'livecd')
+		execute_command_with_env('Install LiveCD utilities', 'emerge -1 livecd-tools', $prtg_cfgrt);
+	if (isset($opts['install_packages'])) {
+		$pkgs=array();
+		foreach (explode(' ', $opts['install_packages']) as $atom) {
+			$pkgs[]=escapeshellarg($atom);
+		}
+		if (count($pkgs) > 1)
+			execute_command_with_env('Check that extra packages will install', 'emerge -p '.implode(' ', $pkgs), $prtg_cfgrt);
+		foreach ($pkgs as $atom)
+			execute_command_with_env('Install extra package '.$atom, 'emerge '.$atom, $prtg_cfgrt);
+	}
+	$imgtype=isset($opts['image_type'])?$opts['image_type']:'tbz2';
+	if ($imgtype == 'tbz2') {
+		execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/image.tar.bz2' -C '$W/image' .");
+		rename($W.'/image.tar.bz2', COMPLETED.'/build-'.$build->id.'.tar.bz2') || throw_exception('rename failed');
+	} elseif ($imgtype == 'tgz') {
+		execute_command('Compress finished image to tar/gz', "tar -p --same-owner -czvf '$W/image.tar.gz' -C '$W/image' .");
+		rename($W.'/image.tar.gz', COMPLETED.'/build-'.$build->id.'.tar.gz') || throw_exception('rename failed');
+	} elseif ($imgtype == 'livecd' || $imgtype == 'installcd') {
+		if (strpos($headers['chost'], 'x86_64') === false)
+			$minimaliso='/home/eitan/soc/install-x86-minimal-20090623.iso';
+		else
+			$minimaliso='/home/eitan/soc/install-amd64-minimal-20090625.iso';
+		execute_command_with_env('Mount minimal CD image', "mount -o loop -t iso9660 '$minimaliso' '$W/tmp'", $prtg_cfgrt); // TODO check if env is necessary (and for umount)
+		execute_command('Copy CD image to writable temp directory', "cp -va '$W/tmp' '$W/cd'");
+		execute_command_with_env('Unmount CD image', "umount '$W/tmp'", $prtg_cfgrt);
+		execute_command('Copy kernel and initrd from CD to image', "cp -va '$W/cd/isolinux/gentoo' '$W/cd/isolinux/gentoo.igz' '$W/image/boot/'");
+		file_put_contents("$W/unsquashfs-files", "/lib64/modules\n\lib\modules\n");
+		execute_command('Copy kernel modules from SquashFS to image', "unsquashfs -i -d '$W/image' -e '$W/unsquashfs-files' '$W/cd/image.squashfs'");
+		if ($imgtype == 'livecd') {
+			rename("$W/cd/image.squashfs", "$W/image.squashfs.old") || debug('Failed to move old SquashFS');
+			execute_command('Compress finished image to squashfs', "mksquashfs '$W/image' '$W/cd/image.squashfs' -noappend -info");
+		} else // Install CD
+			execute_command('Compress finished image to tar/bzip2', "tar -p --same-owner -cjvf '$W/cd/image.tar.bz2' -C '$W/image' .");
+		// TODO port the rest of /usr/lib/catalyst/targets/support/create-iso.sh to support other bootloaders
+		// ISOLINUX bootloader
+		// mkisofs -J -R -l ${mkisofs_zisofs_opts} -V "${clst_iso_volume_id}" -o ${1} -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table ${clst_target_path}
+		execute_command('Create ISO image', "mkisofs -J -R -l -V 'Ingenue Build $build->id' -o '$W/image.iso' -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table '$W/cd'");
+		rename("$W/image.iso", COMPLETED.'/build-'.$build->id.'.iso') || throw_exception('rename failed');
+	} else {
+		throw_exception('invalid image type: '.$imgtype);
+	}
+	return true;
+}
+?>
diff --git a/frontend/wizard/step1.php b/frontend/modules/gentoo/step1.php
index 878fc40..16f04d9 100644
--- a/frontend/wizard/step1.php
+++ b/frontend/modules/gentoo/step1.php
@@ -1,8 +1,8 @@
 <?php
-function wizard_init_step1() {
+function gentoo_init_step1() {
 	return array('title' => 'Step 1 - Choose Profile');
 }
-function wizard_body_step1() {
+function gentoo_body_step1() {
 	global $S;
 	$build=&$S['wizard.build'];
 	//$opts=$build->get_buildopts(); // TODO use this to set selected="selected" on the current profile
@@ -15,7 +15,7 @@ function wizard_body_step1() {
 	}
 	echo '</select><br/>';
 }
-function wizard_process_step1() {
+function gentoo_process_step1() {
 	global $S, $request;
 	$profile=new sql_profile($request['pkgdir']);
 	$profileopt=new sql_buildopt($S['wizard.build']->id, 'profile', $profile->id);
diff --git a/frontend/wizard/step2.php b/frontend/modules/gentoo/step2.php
index 7667b95..4e2907c 100644
--- a/frontend/wizard/step2.php
+++ b/frontend/modules/gentoo/step2.php
@@ -1,8 +1,8 @@
 <?php
-function wizard_init_step2() {
+function gentoo_init_step2() {
 	return array('title' => 'Step 2 - Choose Extra Packages');
 }
-function wizard_body_step2() {
+function gentoo_body_step2() {
 	global $S;
 	$build=&$S['wizard.build'];
 	$opts=$build->get_buildopts();
@@ -22,7 +22,7 @@ function wizard_body_step2() {
 		echo '</ul></li>';
 	}
 }
-function wizard_process_step2() {
+function gentoo_process_step2() {
 	global $S, $request;
 	if (isset($request['extra_packages'])) {
 		$packages=array();
diff --git a/frontend/wizard/step3.php b/frontend/modules/gentoo/step3.php
index 42e5211..f9b83fc 100644
--- a/frontend/wizard/step3.php
+++ b/frontend/modules/gentoo/step3.php
@@ -1,11 +1,11 @@
 <?php
-function wizard_init_step3() {
+function gentoo_init_step3() {
 	return array('title' => 'Step 3 - Image Format');
 }
-function wizard_body_step3() {
+function gentoo_body_step3() {
 	echo 'Image type: <select name="image_type"><option value="tgz">Tar/Gzip</option><option value="tbz2">Tar/Bzip2</option><option value="installcd">Installer CD with Tar/Bzip2 image</option><option value="livecd">LiveCD</option></select><br/>';
 }
-function wizard_process_step3() {
+function gentoo_process_step3() {
 	global $S, $request;
 	if (isset($request['image_type'])) {
 		debug('wizard', 'step3: image type='.$request['image_type']);
diff --git a/frontend/pages/downloadimage.php b/frontend/pages/downloadimage.php
index 7346e6a..91c1161 100644
--- a/frontend/pages/downloadimage.php
+++ b/frontend/pages/downloadimage.php
@@ -34,7 +34,7 @@ function init_downloadimage() {
 	header('Content-Length: '.filesize($S['file']));
 	header('Content-Description: File Transfer');
 	header('Content-Transfer-Encoding: binary');
-	header('Content-Disposition: attachment; filename="ingenue-'.$build->id.$ext);
+	header('Content-Disposition: attachment; filename="'.(isset($build->name) && strlen($build->name)?str_replace('"', '\"', $build->name):'ingenue-'.$build->id).$ext);
 }
 function body_downloadimage() {
 	global $S;
diff --git a/frontend/pages/logview.php b/frontend/pages/logview.php
index 4748a63..5107381 100644
--- a/frontend/pages/logview.php
+++ b/frontend/pages/logview.php
@@ -41,7 +41,8 @@ function body_logview() {
 				$entry=new sql_buildlog_entry($entry);
 //				$text=str_replace(array("\n", "\t"), array("<br/>\n", str_repeat('&nbsp;', 4)), htmlentities($entry->text));
 //				echo '<a name="entry_'.$task->order.'_'.$entry->order.'"'.($entry->stream=='stderr'?' style="color: red" ':'').' title="'.strtoupper($entry->stream).', entry #'.$entry->order.' @ '.date('D j M Y @ H:i:s', $entry->timestamp).' UTC">'.$text.'</a>';
-				echo $ansi->process(nl2br($entry->text));
+				echo $ansi->process(str_replace(array("\n", ' ', "\t"), array("<br/>\n", '&nbsp;', str_repeat('&nbsp;', 8)), $entry->text));
+				// TODO handle tabs properly, move all this into ansi_to_html
 			}
 			echo $ansi->reset(); // Clear any leftover <span>s
 			echo '</div>';
diff --git a/frontend/pages/wizard.php b/frontend/pages/wizard.php
index 18797aa..d8fc9e8 100644
--- a/frontend/pages/wizard.php
+++ b/frontend/pages/wizard.php
@@ -1,6 +1,6 @@
 <?php
 function init_wizard() {
-	global $S, $request;
+	global $S, $request, $conf;
 	if (!isset($S['user'])) {
 		return 'login';
 	}
@@ -10,6 +10,9 @@ function init_wizard() {
 		if ($r->rowCount()) {
 			$S['wizard.build']=new sql_build($r->fetch(PDO::FETCH_ASSOC));
 			$build=&$S['wizard.build'];
+			$opts=$build->get_buildopts();
+			$S['wizard.module']=$opts['frontend_module'];
+			$module=&$S['wizard.module'];
 			if (!preg_match('#^config/step([0-9]+)$#', $build->status, $match)) {
 				debug('wizard', 'build not in config stage (status '.$build->status.') - PANIC!');
 				throw_exception('We haven\'t implemented this yet.');
@@ -19,11 +22,11 @@ function init_wizard() {
 			debug('wizard', 'Build '.$build->id." is at step $step");
 			if (isset($request['wizard_submit'])) {
 				debug('wizard', "Processing step $step");
-				if (!is_file(FRONTEND."/wizard/step$step.php")) {
-					throw_exception('wizard', "Step $step doesn't exist!");
+				if (!is_file(FRONTEND."/modules/$module/step$step.php")) {
+					throw_exception("$modules step $step doesn't exist!");
 				}
-				require_once(FRONTEND."/wizard/step$step.php");
-				$proc='wizard_process_step'.$step;
+				require_once(FRONTEND."/modules/$module/step$step.php");
+				$proc=$module.'_process_step'.$step;
 				if (function_exists($proc)) {
 					if ($proc() === true) {
 						debug('wizard', "Step $step returned <i>true</i> - config finished!");
@@ -34,7 +37,7 @@ function init_wizard() {
 						return array('title' => 'Config Finished');
 					}
 				} else {
-					debug('wizard', 'No processing function for step '.$step);
+					debug('wizard', "No processing function for $wizard step $step");
 				}
 				$build->status='config/step'.++$step;
 				$build->write();
@@ -53,16 +56,28 @@ function init_wizard() {
 			$S['wizard.build']=new sql_build();
 			$S['wizard.build']->init();
 		}
+		$build=&$S['wizard.build'];		
 		$S['wizard.build']->name=$request['name'];
 		$S['wizard.build']->write();
+		$femods=explode(' ', $conf['frontend_modules']);
+		$bemods=explode(' ', $conf['backend_modules']);
+		$femod=isset($request['femod']) && isset($femods[$request['femod']])?$femods[$request['femod']]:$femods[0];
+		$bemod=isset($request['bemod']) && isset($bemods[$request['bemod']])?$bemods[$request['bemod']]:$bemods[0];
+		$beopt=new sql_buildopt($build->id, 'backend_module', $bemod);
+		$feopt=new sql_buildopt($build->id, 'frontend_module', $femod);
+		debug('wizard', "Backend: $bemod; Frontend: $femod");
+		$beopt->write();
+		$feopt->write();
+		$S['wizard.module']=$femod;
+		$module=&$S['wizard.module'];
 		$S['wizard.step']=1;
 	}
 	if (isset($S['wizard.build'], $S['wizard.step'])) {
 		$step=&$S['wizard.step'];
-		if (is_file(FRONTEND."/wizard/step$step.php")) {
-			require_once(FRONTEND.'/wizard/step'.$step.'.php');
+		if (is_file(FRONTEND."/modules/$module/step$step.php")) {
+			require_once(FRONTEND."/modules/$module/step$step.php");
 			$S['title']='Create - Step '.$step;
-			$proc='wizard_init_step'.$step;
+			$proc=$module.'_init_step'.$step;
 			if (function_exists($proc)) {
 				return $proc();
 			} else {
@@ -70,31 +85,48 @@ function init_wizard() {
 				return;
 			}
 		} else {
-			throw_exception("Step $step doesn't exist");
+			throw_exception("$module step $step doesn't exist");
 		}
 	}
 	return array('title' => 'Create');
 }
 function body_wizard() {
-	global $S;
+	global $S, $conf;
 	if (isset($S['wizard.build'])) {
 		if (isset($S['wizard.done'])) {
 			echo '<h3>Config finished!</h3><p>Check your build\'s status <a href="'.url('logs/'.$S['wizard.build']->id).'">here</a></p>';
 		} else {
 			$build=&$S['wizard.build'];
+			$module=&$S['wizard.module'];
 			$step=&$S['wizard.step'];
 			echo '<h3>'.$S['title'].'</h3>';
 			echo '<form action="'.url('create/'.$build->id).'" method="post">';
-			require_once(FRONTEND."/wizard/step$step.php");
-			$proc='wizard_body_step'.$step;
+			require_once(FRONTEND."/modules/$module/step$step.php");
+			$proc=$module.'_body_step'.$step;
 			if (!function_exists($proc)) {
-				throw_exception("Body function doesn't exist for wizard step $step");
+				throw_exception("Body function doesn't exist for $module step $step");
 			}
 			$proc();
 			echo '<input type="submit" name="wizard_submit" value="Continue" /></form>';
 		}
 	} else {
 		echo '<form action="'.url('create').'" method="post"><h3>Request an image built</h3>Name of your build (optional): <input name="name" /><br/>';
+		$femods=explode(' ', $conf['frontend_modules']);
+		$bemods=explode(' ', $conf['backend_modules']);
+		if (count($femods) > 1) {
+			echo 'Frontend module: <select name="femod">';
+			$i=0;
+			foreach ($femods as $mod)
+				echo '<option value="'.$i++."\">$mod</option>";
+			echo '</select><br/>';
+		}
+		if (count($bemods) > 1) {
+			echo 'Backend module: <select name="bemod">';
+			$i=0;
+			foreach ($bemods as $mod)
+				echo '<option value="'.$i++."\">$mod</option>";
+			echo '</select><br/>';
+		}
 		echo '<input type="submit" name="init" value="Start" /></form>';
 	}
 }
diff --git a/shared/classes/task.php b/shared/classes/task.php
index 796864f..569d0fb 100644
--- a/shared/classes/task.php
+++ b/shared/classes/task.php
@@ -76,7 +76,7 @@ class sql_task extends sql_row_obj {
 		$html.='</div></div>';
 		return $html;
 	}
-	function execute($fatal=true, $path=null, $env=null) {
+	function execute($path=null, $env=null) {
 		if (!isset($this->command, $this->build, $this->order)) {
 			throw_exception('$this->command, $this->build, or $this->order not set');
 		} elseif (isset($this->start)) {
@@ -142,37 +142,7 @@ class sql_task extends sql_row_obj {
 			$this->exit=proc_close($p);
 		}
 		$this->write();
-		if ($this->exit == 0)
-			log_msg(color('[success]', 'green'));
-		elseif ($this->exit > 0) {
-			log_msg(color('[exit code '.$this->exit.']', 'red'));
-			if ($fatal)
-				throw_exception($this->command.' returned with exit status '.$this->exit);
-		} elseif ($this->exit == -128) {
-			log_msg(color('[received unknown signal]', 'red'));
-			if ($fatal)
-				throw_exception($this->command.' received an unknown signal');
-		} else {
-			log_msg(color('[received signal '.-$this->exit.']', 'red'));
-			if ($fatal)
-				throw_exception($this->command.' received signal '.-$this->exit);
-		}
 		return $this->exit;
 	}
-	static function execute_command($description, $command, $build, $fatal=true, $path=null, $env=null) {
-		global $task;
-		// TODO this won't work once we have internal tasks too - we need to modify the build class to track its own task IDs
-		static $buildid=null;
-		static $order=0;
-		if ($build->id !== $buildid) {
-			$buildid=$build->id;
-			$order=0;
-		}
-		$task=new sql_task($build->id, $order, null, $description, $command);
-		$result=$task->execute($fatal, $path, $env);
-		$order++;
-		unset($task);
-		return $result;
-	}
 }
 ?>
diff --git a/shared/config.php b/shared/config.php
index 53ab10a..c8f1179 100644
--- a/shared/config.php
+++ b/shared/config.php
@@ -6,6 +6,8 @@ $conf['sqlpass']='socpassword'; // MySQL password
 $conf['sqldb']='soc'; // MySQL database
 $conf['debug']=true; // Whether to print debugging information
 $conf['cache']=true; // Whether to enable built-in caching
+$conf['backend_modules']='gentoo_portage gentoo_catalyst'; // Space-separated list of backend modules to offer the user
+$conf['frontend_modules']='gentoo'; // Space-separated list of frontend modules to offer the user
 $conf['cookiename']='ingenueid'; // Name of the cookie to send for keeping sessions
 $conf['sessionlength']=1814400; // Time in seconds before sessions are purged
 $conf['timezone']=10800; // Time difference in seconds between UTC and the default timezone
@@ -16,7 +18,7 @@ $conf['pkgdir_root']='/home/eitan/soc/tinderbox'; // The directory to recursivel
 $conf['emerge_default_opts']='-t -v -K --color=y --root-deps=rdeps'; // DON'T CHANGE UNLESS YOU KNOW WHAT YOU'RE DOING
 $conf['portdir']='/usr/portage'; // The directory conatining the portage tree to use (/usr/portage unless you have a reason to think otherwise)
 $conf['logview_max']=1000; // The maximum number of log entries shown on one page (1000 is a good start)
-$conf['split_setip']=true; // Whether the frontend and backend are running on different hosts
+$conf['split_setup']=true; // Whether the frontend and backend are running on different hosts
 $conf['frontend_location']='http://soc'; // The base address of the frontend installation (for use by the backend)
 $conf['backend_name']='red'; // A name or other way of identifying this backend as opposed to other backends working for the same frontend TODO use gethostname() by default in 5.3.0
 ?>
diff --git a/todo b/todo
index 35989d1..8478a6e 100644
--- a/todo
+++ b/todo
@@ -1,12 +1,8 @@
-CLI interface to backend? (Or cli interface to frontend...)
-Find kernels
 Have backend handle builds that it finds to already be running (break in to steps and store current status)
-Make frontend package adding a little more user-friendly (ajax or client-side js search)
-Restructure the backend for modularity
-Make frontend and backend actually work like they're using one of several modules
+*** Make frontend package adding a little more user-friendly (ajax or client-side js search) ***
 Write a live git ebuild
 Write an AJAX-based self-updating status viewer
-Add logging besides just commands (restructure the buildlogs table)
+Add logging besides just commands
 Make backend do a dummy run through and queue all commands and other tasks, then execute them (for better status handling, easier debugging, etc.)
 Have builds and tasks not give links to logs if we're already viewing the logs
 Either make task status a TEXT or stop putting command name in the status (via thrown exception) - we can fetch this later anyway - just store the task id that failed (or use the last task)
@@ -20,8 +16,12 @@ Separate variables we got from the URL from the rest, stop using $request, inste
 Get rid of backend use of echo, debug, etc. and do all logging in the db
 Support ~arch installation or remove it from listings
 Completely plan out how frontend modules should function - each step needs to report if it finished successfully (required values)
-Change over wizard to work on configurations, not actual builds
+*** Change over wizard to work on configurations, not actual builds ***
 See if I can figure any way to use classes instead of funny named functions (namespaces in 5.3.0 perhaps) - also needs to have a depend of some sort so we can make a quick-config page that doesn't show things that we don't have enough info for yet (use AJAX to show them when we're ready)
 Allow backend to define bail-out functions to call when it dies (things like unmounting the ISO it was copying)
 Add STDERR (maybe STDOUT) only option to log viewer
-Stop directly accessing PKGDIR from the frontend - put whatever info we need in the database and write a script for updating it from the backend
+*** Make frontend/backend split possible ***
+	Stop directly accessing PKGDIR from the frontend - put whatever info we need in the database and write a script for updating it from the backend
+	Have backend upload finished images to frontend
+Add options viewer - part of config process should be read-only version
+Cache the image after emerge system to save time