#!/usr/bin/python # -*- Mode: Python -*- # vi:si:et:sw=4:sts=4:ts=4 # # mach - make a chroot # # script to set up a chroot using packages and optionally perform builds in it import sys, os, stat import getopt, string, commands, urllib, shutil, time, fcntl, re import rpm # FIXME: find out what rpm version we need ! import random # spinner theming madness import tempfile # for tempdir for spec extraction import glob # for globbing paths import grp # for finding out the mach gid # make sure True and False constants exist if not hasattr (__builtins__, 'True'): __builtins__.True = (1 == 1) __builtins__.False = (1 != 1) # usage and help usage = 'usage: mach [mach-options] command [command-options-and-arguments]' help = '''Make a chroot and do stuff in it. Mach options: -v, --version print mach version -r, --root=ROOT specify a root to work in -d, --debug turn on debugging -f, --force override root lock -k, --keep do not revert buildroot to initial package list -s, --sign sign packages after build -c, --collect collect packages and relevant files to . -q, --quiet no spinners, minimal output --release=(tag) add given tag to release tag --no-scripts don\'t run post build scripts --canonify just output the canonical root name and exit Commands: build build from (set of) .spec file(s) (passes options given to rpmbuild) chroot chroot into root clean totally cleans the root rebuild rebuild a (set of) .src.rpm(s) (passes options given to rpmbuild) setup sets up the root for a particular purpose prep preparation before package install minimal minimal set of rpms (bash, glibc) base base set of rpms for minimal functionality build everything to build rpms status show status of mach roots unlock override root lock yum run host\'s yum on target root ''' # list of allowed commands allowed_commands = ('build', 'chroot', 'clean', 'rebuild', 'rpm', 'setup', 'status', 'unlock', 'yum') # read default configuration config = { 'force': 0, 'check': 0, 'sign': 0, 'collect': 0, 'release': 0, 'quiet': 0, 'scripts': 1, 'defaultroot': 'fedora-1-i386', # FIXME: based on configure's distro check 'chroot': '/usr/local/sbin/mach-helper chroot', 'mount': '/usr/local/sbin/mach-helper mount', 'rpm': '/usr/local/sbin/mach-helper rpm', 'umount': '/usr/local/sbin/mach-helper umount', 'rm': '/usr/local/sbin/mach-helper rm', 'mknod': '/usr/local/sbin/mach-helper mknod', 'yum': '/usr/local/sbin/mach-helper yum', 'runuser': '/usr/local/sbin/runuser', 'installer': 'yum', # root_*: commands run inside the build root 'root_rpmbuild': 'rpmbuild', } DEBUG = 0 # read system-wide configuration file = '/usr/local/etc/mach/conf' try: execfile (file) except: sys.stderr.write ("Could not read config file %s: %s\n" % (file, sys.exc_info()[1])) sys.exit (1) # read dist information # clear stuff yumsources = {} packages = {} groups = {} sourceslist = {} aliases = {} # first, get locations file = '/usr/local/etc/mach/location' try: execfile (file) except: sys.stderr.write ("Could not read location file %s: %s\n" % (file, sys.exc_info()[1])) sys.exit (1) # next, get all of the dist.d files dists = os.listdir ('/usr/local/etc/mach/dist.d') dists.sort() for file in dists: # skip various files, backups etc if re.search ('^[\.#]|(\.rpm(orig|new|save)|~)$', file): continue file = os.path.join ('/usr/local/etc', 'mach', 'dist.d', file) try: execfile (file) except: sys.stderr.write ("Could not read dist file %s: %s\n" % (file, sys.exc_info()[1])) sys.exit (1) # read user configuration file = os.path.join (os.environ['HOME'], '.machrc') try: execfile (file) except IOError: pass # rpm -q format, users may have customized %_query_all_fmt in ~/.rpmmacros... qfmt = '%{name}-%{version}-%{release}\n' # build user/group inside chroot builduser = 'machbuild' buildgroup = 'machbuild' ### objects class Spec: class Error(Exception): pass def __init__ (self, path, options = ""): self.path = path self.content = [] self.vars = {} self.options = options debug ("Spec.__init__: opening spec %s" % path) # check if the spec file has a %prep section command = "grep '%%prep' %s" % path (status, output) = commands.getstatusoutput (command) if os.WIFEXITED (status): retval = os.WEXITSTATUS (status) if (retval != 0): raise self.Error, "spec file %s does not have a %%prep section" % self.path # check for syntax errors in the spec file first command = "rpmbuild -bp --nodeps --force %s --define '__spec_prep_pre exit 0' --define 'setup :' %s" % (options, path) debug ("Spec:__init__: command: %s" % command) (status, output) = commands.getstatusoutput (command) if os.WIFEXITED (status): retval = os.WEXITSTATUS (status) if (retval != 0): raise self.Error, "spec file %s fails to parse:\n%s" % (self.path, output) self.content = open (self.path).readlines () # expand tags using rpmbuild because results can depend on # build options for which in ['name', 'version', 'release']: command = "rpmbuild -bp --nodeps --force %s --define '__spec_prep_pre echo %%%s; exit 0' --define 'setup :' %s | tail -n 1" % (options, which, path) debug ("Spec:__init__: command: %s" % command) value = commands.getoutput (command) self.vars[which] = value # scan for defines in the spec file debug ("scanning spec file for %define's") matchstr = re.compile ("^\s*%define\s+(\S*)\s+(\S*)$") for line in self.content: match = matchstr.search (line) if match: tag = match.expand ("\\1") value = match.expand ("\\2") debug ("Found define of %s to %s" % (tag, value)) self.vars[tag] = value # now loop over all of the tag values and expand them until we're done # FIXME: an anti-loop here would be nice, wouldn't it ? # FIXME: for now, we do a single pass loop # FIXME: also realize this will bum us out with multiple %{...}, so # FIXME: in the future make this non-greedy or something matchstr = re.compile ("^(.*)%{(.*)}(.*)$") for tag in self.vars.keys (): try: match = matchstr.search (self.vars[tag]) except TypeError: sys.stderr.write ("Error on searching for tag %s with value %s" % (tag, self.vars[tag])) if match: expand = match.expand ("\\2") debug ("Need to expand tag %s inside tag %s (%s)" % (expand, tag, self.vars[tag])) if expand in self.vars.keys (): debug ("Expanding it to %s" % self.vars[expand]) front = match.expand ("\\1") back = match.expand ("\\3") self.vars[tag] = front + self.vars[expand] + back debug ("Expanded complete line to %s" % self.vars[tag]) def tag (self, which): "Get the given (unique) tag from the spec file as a string" # if we already have it, return it if self.vars.has_key (which): return self.vars[which] # if we don't, look for it in the file matchstr = re.compile ("^" + which + r"\d*\s*:\s*(.*)", re.I) for line in self.content: match = matchstr.search (line) if match: value = match.expand ("\\1") self.vars[which] = value return value sys.stderr.write ("WARNING: didn't find tag %s in spec file\n" % which) return None def tags (self, which): "Get all given tags from the spec file as a list" result = [] matchstr = re.compile ("^" + which + r"\d*:\s*(.*)", re.I) for line in self.content: match = matchstr.search (line) if match: value = match.expand ("\\1") result.append (value) return result # this is done quick and dirty, could be optimized more def expand (self, expression): "expand a line from a spec file using vars from the spec file" for variable in self.vars.keys (): expression = string.replace (expression, "%{" + variable + "}", self.vars[variable]) #print "DEBUG: expanded to %s" % expression return expression def nvr (self): "return name, version, release triplet" return (self.vars['name'], self.vars['version'], self.vars['release']) def _sourcespatches (self, which, args = ""): # get a dict of Source and Patch lines, given quoted options to build # as args result = {} matchstr = re.compile ("^" + which + r"(\d*):\s*(.*)") macro = "" if which == "Source": macro = "SOURCEURL" if which == "Patch": macro = "PATCHURL" for line in self.content: match = matchstr.search (line) if match: number = match.expand ("\\1") value = match.expand ("\\2") tag = which + number if number == "": number = "0" command = "rpmbuild -bp --nodeps --force --define '__spec_prep_pre echo %%%s%s; exit 0' --define 'setup :' %s %s | tail -n 1" % (macro, number, args, self.path) debug (command) result[tag] = commands.getoutput (command) return result def sources (self): # parse the spec file for all Source: files return self._sourcespatches ("Source", self.options) def patches (self): # parse the spec file for all Source: files return self._sourcespatches ("Patch", self.options) ### abstract object wrapping a src rpm class SRPM: class Error(Exception): pass def __init__ (self, path, options = ""): self.path = path self.header = self._header () self.specname = self._specname () self.options = options # ignore signatures def _header (self): "get the rpm header from the src.rpm" debug ("Getting RPM header from %s" % self.path) ts = rpm.TransactionSet ("", (rpm._RPMVSF_NOSIGNATURES)) fd = os.open (self.path, os.O_RDONLY) try: header = ts.hdrFromFdno (fd) except: print "a whoopsie occurred." if not header: raise self.Error, "%s doesn't look like a .src.rpm" % self.path return header def _specname (self): "return the name of the specfile inside this src.rpm" # FIXME: we found an rhas spec that had something.spec.in as the spec # file, so find a better way to get the spec file name name = commands.getoutput ('rpm -qlp %s 2> /dev/null | grep \.spec$' % self.path) if not name: name = commands.getoutput ('rpm -qlp %s 2> /dev/null | grep \.spec.*$' % self.path) if not name: raise self.Error, "Didn't find spec file in %s" % self.path debug ("SRPM.specname: %s" % name) return name def BuildRequires (self): "return a list of BuildRequires for this src.rpm" buildreqs = self.header[rpm.RPMTAG_REQUIRENAME] debug ("BuildRequires from header: %s" % buildreqs) # remove non-packagey things from buildreqs, like autodeps # currently removes everything containing / and all rpmlib() deps if buildreqs: return filter (lambda x: not (re.search("rpmlib", x)), buildreqs) else: return [] def results (self): "returns a list of packages that can be built from this src.rpm" # FIXME: this extracts the spec file out of the src.rpm and # puts it in a temp dir; is this the cleanest option ? debug ("Getting possible result packages from src.rpm %s" % self.path) path = self.path specname = self.specname tmpdir = tempfile.mktemp () ensure_dir (tmpdir) tmp_remove_cmd = 'rm -rf %s' % tmpdir cmd = 'cd %s && rpm2cpio %s \ | cpio -id --no-absolute-filenames %s \ > /dev/null 2>&1' % (tmpdir, path, specname) os.system (cmd) spectmp = os.path.join (tmpdir, self.specname) if not os.path.exists (spectmp): raise self.Error, "didn't find spec file in %s" % self.path cmd = "rpm -q --specfile %s" % spectmp output = commands.getoutput (cmd) debug ("output of rpm -q --specfile: \n%s" % output) try: spec = Spec (spectmp, self.options) except Spec.Error, message: os.system (tmp_remove_cmd) raise self.Error, message (n, v, r) = spec.nvr () # strip v and r off of output result = [] for line in output.split ("\n"): result.append (line.replace ('-%s-%s' % (v, r), '')) debug ("results of srpm: %s" % result) os.system (tmp_remove_cmd) return result ### abstract object acting on the root # FIXME: we can subclass this based on actual method to install stuff class Root: # exceptions that can be raised class Locked(Exception): pass class Error(Exception): pass class ReturnValue(Exception): pass # read in configuration from disk when creating a root def __init__ (self, config): self.config = config self.hooks = self.config.get ('hooks', {}) self.root = self.config['root'] self.rootdir = get_config_dir (config, 'root') self.resultdir = get_config_dir (config, 'result') self.statedir = get_config_dir (self.config, 'state') self.tmpdir = get_config_dir (self.config, 'tmp') self.packagesdir = os.path.join (config['dirs']['packages'], config['packages']['dir']) # small helper functions def lock (self): "Lock this root for operations" ensure_dir (self.statedir) debug ("locking root") lockpath = os.path.join (self.statedir, 'lock') if os.path.exists (lockpath): if not self.config['force']: raise self.Locked else: print 'warning: overriding lock on root %s' % self.root try: lockfile = open (lockpath, 'w') except: raise self.Error, "error: can't create lock file for %s !\n" % self.root lockfile.close () def unlock (self, args = []): "Unlock this root for operations" debug ("unlocking root") lockpath = os.path.join (self.statedir, 'lock') if os.path.exists (lockpath): os.remove (lockpath) def get_state (self, which): "Returns None if the state file doesn't exist, and contents if it does" path = os.path.join (self.statedir, which) if os.path.exists (path): return open (path, 'r').readlines () else: return None # default here assures that state files are never empty def set_state (self, which, content = ""): "Creates the state file and fill it with given contents." #"raises Error if it already existed." if not content: content = which path = os.path.join (self.statedir, which) #if os.path.exists (path): # raise self.Error, "State file %s exists already" % path debug ('outputting state') debug (content) debug ('outputted state') open (path, 'w').write (content) def progress (self): "returns True if progress information should be shown" if self.config['quiet']: return False return True def stdout (self, message): #FIXME: check verbosity here "outputs this message to stdout" sys.stdout.write (message) sys.stdout.flush () def mount (self): "mount proc into chroot" # FIXME: the next three lines are workaround code for when not having # proc # FIXME: rh 72's mount needs /proc/mounts file, let's give it one #open (os.path.join (self.rootdir, 'proc', 'mounts'), 'w').close () #return ensure_dir (os.path.join (self.rootdir, 'proc')) # first umount for completeness; you never know self.umount () debug ("mounting proc") file = open (os.path.join (self.statedir, 'mount'), "a+") command = '%s -t proc proc %s/proc' % (self.config['mount'], self.rootdir) debug (command) os.system (command) file.write ('%s/proc\n' % self.rootdir) file.close () # umount all mounted paths def umount (self): if not self.get_state: return mountfile = os.path.join (self.statedir, 'mount') if not os.path.exists (mountfile): return debug ('cat %s | xargs %s' % (mountfile, self.config['umount'])) os.system ('cat %s | xargs %s' % (mountfile, self.config['umount'])) os.remove (mountfile) # recreate a given file listed in config['files'] def config_recreate (self, filename): debug ("Recreating %s from config" % filename) if self.config['files'].has_key (filename): new = self.rootdir + filename if os.path.exists (new): return ensure_dir (os.path.dirname (self.rootdir + filename)) tmpname = os.path.join ('tmp', os.path.basename (filename)) file = open (os.path.join (self.rootdir, tmpname), 'w') file.write (config['files'][filename]) file.close () try: os.rename (os.path.join (self.rootdir, tmpname), new) except OSError: raise self.Error, "Could not create %s" % new def _splitarg (self, matcher, args): "only used by splitargspecs and splitargsrcrpms" "put everything from args matching matcher on result" "put everything from args not matching matcher on other" "return (result, other)" result = [] other = [] match = re.compile (matcher, re.I) for arg in args: if match.search (arg): result.append (arg) else: other.append (arg) return (other, result) # seek through the argument list for specfiles def splitargspecs (self, args): return self._splitarg ('\.spec$', args) # seek through the argument list for srpms def splitargsrpms (self, args): return self._splitarg ('\.src\.rpm$', args) # if message is specified, it will be printed, and newline-terminated # properly if no message specified, then it's up to caller def installer_get (self, args, message = None, progress = False, interactive = False): "run yum (args) from outside the root on the root" installer = self.config[self.config['installer']] args = string.join(args) # args must be a list conf = os.path.join(self.statedir, 'yum.conf') extras = '--installroot %s' % self.rootdir command = "%s %s -c %s %s" % (installer, extras, conf, args) debug("installer_get: command %s" % command) if interactive: return os.system(command) if message: self.stdout(message) if progress: self.stdout(' ...') (status, output) = do_with_output(command, progress) if message and not progress: self.stdout('\n') if status != 0: raise self.Error, "Could not run" % command def group_installer (self, groups, message = None, progress = False, interactive = False): "run yum groupinstall (arg) from outside the root on the root" installer = self.config[self.config['installer']] conf = os.path.join(self.statedir, 'yum.conf') extras = '--installroot %s' % self.rootdir command = "%s %s -c %s groupinstall %s" % (installer, extras, conf, groups) debug("group_installer: command %s" % command) if interactive: return os.system (command) if message: self.stdout (message) if progress: self.stdout (' ...') (status, output) = do_with_output (command, progress) if message and not progress: self.stdout ('\n') if status != 0: raise self.Error, "Could not get %s" % groups def rpm (self, args, progress = False): "run rpm (arg) from outside the root on the root, returns output" command = "%s --root %s %s" % (self.config['rpm'], self.rootdir, join_quoted (args)) # /proc needs to be mounted because we might trigger RPM scripts # in the chroot and FC3's ld.so seems to open /proc/self/attr/exec, # making the scripts fail with a return code of -1 if /proc is not there self.mount () (status, output) = do_with_output (command, progress) self.umount () if status != 0: raise self.Error, "Could not rpm %s" % join_quoted (args) return output def yum(self, args, progress = False, interactive = True): """run yum (args) from outside the root on the root""" installer = self.config[self.config['installer']] args = string.join(args) # args must be a list conf = os.path.join(self.statedir, 'yum.conf') extras = '--installroot %s' % self.rootdir command = "%s %s -c %s %s" % (installer, extras, conf, args) debug("yum: command %s" % command) self.mount () if interactive: return os.system(command) if progress: self.stdout(' ...') (status, output) = do_with_output(command, progress) self.umount () if status != 0: raise self.Error, "Could not run" % command # main command executer: # - executes on host # - direct execution with std's connected or output captured (FIXME) # - show or don't show progress meter # FIXME: user ? def do (self, command, progress = True, user = ""): "execute given command" debug ("Executing %s" % command) if progress: return do_progress (command) else: (status, output) = commands.getstatusoutput (command) if os.WIFEXITED (status): retval = os.WEXITSTATUS (status) if (retval != 0): raise self.ReturnValue, (retval, output) # FIXME: we send back retvals with a raise now, do the same # in do_progress return (retval, output) def do_chroot (self, command, message = None, progress = False, user = "", fatal = False): # if you call do_chroot, then do_chroot is responsible for printing # \n "execute given command in root" # HACK: FIXME: # if the cmd already contains -c " as a sequence, then don't wrap # it in /bin/bash cmd = "" if string.find (command, '-c "') > -1: cmd = "%s %s %s" % (config['chroot'], self.rootdir, command) else: # we use double quotes to protect the commandline since # we use single quotes to protect the args in command cmd = "%s %s %s - root -c \"%s\"" % (config['chroot'], self.rootdir, self.config['runuser'], command) # be quiet if we're asked to be so if (self.progress () == False): progress = False if message: self.stdout (message) if progress: self.stdout (' ...') (ret, output) = self.do (cmd, progress) if message and not progress: self.stdout ('\n') if (ret != 0): if fatal: # FIXME. what about unmounting ? sys.stderr.write ("Non-zero return value %d on executing %s\n" % (ret, cmd)) exit (ret) return (ret, output) # check the given package list file # remove all packages that aren't in this list file def check_package_list (self, listfile): "check the given package list file and remove packages not listed" debug ("checking package list against snapshot") if self.config.has_key ('keep'): print "Not removing packages from root." return root = self.rootdir cmd = "rpm --root %s -qa --qf '%s' \ | diff - %s > /dev/null 2>&1" % (root, qfmt, listfile) debug ("running " + cmd) if os.system (cmd) == 0: # no differences return True cmd = "rpm --root %s -qa --qf '%s' \ | grep -v -f %s - \ | tr \"\n\" \" \"" % (root, qfmt, listfile) packages = commands.getoutput(cmd) pkglist = packages.split() if not pkglist: return True sys.stdout.write ("Removing %d packages ..." % len(pkglist)) commandlist=['remove'] commandlist.extend(pkglist) try: self.installer_get(commandlist, progress = self.progress()) except self.Error: cmd = string.join(commandlist) print "Error removing files from this chroot command was: \n %s" % cmd return True def build (self, args): "build from a set of spec files by packaging them up as src.rpm and " "passing them to rebuild" if not args: raise self.Error, 'Please supply .spec files to build' # check if we can build here yet if not self.get_state ('build'): self.setup (['build', ]) self.lock () srpms = [] # resulting set of srpms # separate options to rpmbuild from specfiles (options, specs) = self.splitargspecs (args) debug ("Build options to %s: %s" % (self.config['root_rpmbuild'], options)) # check if the spec files exist and if we can parse the files necessary for specfile in specs: if not os.path.exists (specfile): raise self.Error, "spec %s does not exist !" % specfile self.stdout ("Building .src.rpm from %s\n" % os.path.basename (specfile)) optstring = join_quoted (options) debug ("build: quoted optstring: %s" % optstring) try: spec = Spec (specfile, optstring) except Spec.Error, message: raise self.Error, message (n, v, r) = spec.nvr () # download all referenced files (Sources, Patches) downloads = spec.sources ().values () + spec.patches ().values () # which paths will we check for already existing files ? # FIXME: add a temp stuff path somewhere instead of using current tmppath = "%s/%s-%s-%s" % (self.tmpdir, n, v, r) ensure_dir (tmppath) # check temporary dir, current dir, and dir where specfile lives paths = [tmppath, '.', os.path.dirname (specfile)] # check if the files mentioned aren't already on-disk files = [] debug ("build: files to download: %s" % downloads) debug ("build: paths to check: %s" % paths) if downloads: for download in downloads: found = 0 filename = os.path.basename (download) for path in paths: filepath = os.path.join (path, filename) if os.path.exists (filepath): self.stdout ("Using %s\n" % filepath) files.append (filepath) found = 1 if not found: filepath = os.path.join (tmppath, filename) try: urlgrab (download, filepath) except: raise self.Error, "Could not download %s !" % download files.append (filepath) self.stdout ("Using %s\n" % filepath) # copy all necessary files to /tmp in root, then chroot to mv them for file in files: debug ("Getting file %s into SOURCES" % file) # sanity check the given file using "file" # crap, this completely hangs on fedora core 0.94 :( # command = "file -b -i -z %s" % file # if file[-7:] == ".tar.gz": # line = os.popen (command).readline ().rstrip () # if not line[:17] == 'application/x-tar': # raise self.Error, "file doesn't think %s is a tar.gz !" % file # # didn't seem to work on MPlayer 0.91 tar.bz2 :( #if file[-8:] == ".tar.bz2": # line = os.popen (command).readline () # if not line == 'application/x-tar gnu (application/octet-stream)': # raise self.Error, "file doesn't think %s is a tar.bz2 !" % file shutil.copy2 (file, os.path.join (self.rootdir, 'tmp')) self.do_chroot ("cd / && mv %s %s" % ( os.path.join ('tmp', os.path.basename (file)), os.path.join ('usr', 'src', 'rpm', 'SOURCES'))) # remove the spec if it existed, then copy self.do_chroot ("cd / && rm -f %s" % ( os.path.join ('tmp', os.path.basename (specfile)))) shutil.copy2 (specfile, os.path.join (self.rootdir, 'tmp')) self.do_chroot ("cd / && mv %s %s" % ( os.path.join ('tmp', os.path.basename (specfile)), os.path.join ('usr', 'src', 'rpm', 'SPECS'))) # fix ownership on all of these self.do_chroot ("cd /usr/src/rpm && chown -R %s:%s *" % (builduser, buildgroup)) try: self.do_chroot ("cd / && %s %s -bs --nodeps %s" \ % (self.config['root_rpmbuild'], optstring, os.path.join ('usr', 'src', 'rpm', 'SPECS', os.path.basename (specfile))), "Creating .src.rpm", True) except self.ReturnValue: raise self.Error, 'could not build .src.rpm' (n, v, r) = spec.nvr () srpmname = "%s-%s-%s.src.rpm" % (n, v, r) debug ("DEBUG: resulting srpm: %s" % srpmname) srpmfile = os.path.join (self.rootdir, 'usr', 'src', 'rpm', 'SRPMS', srpmname) if not os.path.exists (srpmfile): raise self.Error, '%s not built' % srpmfile shutil.copy2 (self.rootdir + '/usr/src/rpm/SRPMS/' + srpmname, tmppath) srpms.append (os.path.join (tmppath, srpmname)) # ready to build them all print "Rebuilding generated .src.rpm's: \n- %s" % string.join (srpms, "\n- ") self.unlock () self.rebuild (options + srpms) def _bruteclean (self): "brutishly clean out the root by using mach-helper rm -rfv" command = "%s -rfv %s" % (self.config['rm'], self.rootdir) self.umount () sys.stdout.write ("Brutishly removing %s ..." % self.rootdir) (status, output) = do_with_output (command, True) if status != 0: raise self.Error, "Could not remove %s" % self.rootdir def clean (self, args = []): "clean out the root" # does the root still exist ? if not os.path.exists (self.rootdir): return self.umount () # remove all state info debug ("Removing statedir %s" % self.statedir) os.system ("rm -rf %s" % self.statedir) # FIXME: check if the dir isn't already clean yet if not os.path.exists (self.rootdir + '/bin/rm'): self._bruteclean () return # get a list of files and dirs in the root files = string.join (os.listdir (self.rootdir)) self.do_chroot ('cd / && rm -rfv %s' % files, "Cleaning out root", True) try: os.rmdir (self.rootdir) except OSError: # assume we couldn't delete because not empty and bring out the # big guns self._bruteclean () # chroot into root def chroot (self, args): self.mount () # a minimal system might not have su/runuser, yikes if not os.path.exists (os.path.join (self.rootdir, 'bin', self.config['runuser'])): if args: print "No %s in root, can't execute %s" % ( self.config['runuser'], join_quoted (args)) return else: cmd = "%s %s %s" % (self.config['chroot'], self.rootdir, join_quoted (args)) print "Entering %s, type exit to leave." % self.rootdir elif args: cmd = "%s %s %s - root -c '%s'" % (self.config['chroot'], self.rootdir, self.config['runuser'], join_quoted (args)) else: cmd = "%s %s %s - root" % (self.config['chroot'], self.rootdir, self.config['runuser']) print "Entering %s, type exit to leave." % self.rootdir debug ("running %s" % cmd) os.system (cmd) self.umount () def setup (self, args): "Set up the root to a given target" target = '' if args: target = args[0] else: target = 'base' # default to build targets = ('prep', 'minimal', 'base', 'build') # see if something real given to setup if target not in targets: raise self.Error, "don't know how to set up %s" % target self.lock () # try each of the targets in turn for which in targets: debug ("setting up target %s" % which) method = '_setup_' + which if method not in Root.__dict__.keys (): raise self.Error, "no _setup_%s method" % which Root.__dict__[method] (self) if target == which: break self.unlock () def rebuild (self, args): if not args: raise self.Error, 'Please supply .src.rpm files to build' (options, paths) = self.splitargsrpms (args) self.setup (['build', ]) self.lock () # get pkgs and collect info pkgs = {} for path in paths: # resolve path to basename (without dirs) srpmname = os.path.basename (path) newpath = os.path.join (self.rootdir, 'tmp', srpmname) try: urlgrab (path, newpath) except: raise self.Error, "Can't find %s" % path srpm = SRPM (newpath, join_quoted (options)) spec = srpm.specname buildreqs = srpm.BuildRequires () name = srpm.header[rpm.RPMTAG_NAME] pkgs[name] = {} pkgs[name]['path'] = newpath pkgs[name]['buildreqs'] = buildreqs pkgs[name]['srpm'] = srpm pkgs[name]['srpmname'] = srpmname pkgs[name]['header'] = srpm.header pkgs[name]['spec'] = spec # now figure out build order based on buildrequires of each pkg if len(pkgs) > 1: deps = [] # list of pkg, dep pairs for pkg in pkgs.keys (): debug ("Processing %s for build order" % pkgs[pkg]['srpmname']) # get results from this package results = pkgs[pkg]['srpm'].results () # figure out the build requirements and add # pkg -> (results) and buildreq -> (results) to chain for result in results: if result != pkg: debug ("adding %s depends on pkg %s" % (result, pkg)) deps.append ((result, pkg)) if not pkgs[pkg]['buildreqs']: continue for buildreq in pkgs[pkg]['buildreqs']: if (result == buildreq): sys.stderr.write ("WARNING: package %s BuildRequires: itself, packaging error !" % result) else: debug ("adding %s depends on buildreq %s" % (result, buildreq)) deps.append ((result, buildreq)) debug ("topological input: ") debug (deps) debug ("topological output: ") try: sorted = topological_sort (deps) except CycleError: sys.stderr.write (''' ERROR: You've hit a dependency loop in the packages you are trying to build. This means that the BuildRequires: of the various packages form a loop. You can work around this by breaking the loop in two pieces, and first building the one chain, then the other, then the one again based on the other's new packages.''') exit (config) sorted.reverse () debug ("order %s" % sorted) # now scrub packages not up for build from order order = [] for pkg in sorted: if pkg in pkgs.keys (): order.append (pkg) debug ("(without packages up for build) order %s" % order) # now make sure all packages given are up for build for pkg in pkgs.keys (): if pkg not in order: order.append (pkg) debug ("(with all packages up for build) order %s" % order) else: order = pkgs # now build self.mount () resultdirs = [] # will contain each of the resultdir's for this build # loop over all package names for name in order: srpmname = pkgs[name]['srpmname'] # remove packages if not asked to keep self._check_revert_build () # install buildrequires if there are debug ("Building %s with package stuff %s" % (name, pkgs[name].keys ())) print "Building source rpm %s" % srpmname if pkgs[name].has_key ('buildreqs') and pkgs[name]['buildreqs']: #FIXME: we filter out packages here that can have multiple #versions, and expect the user to install them manually #this works around kernel and kernel-source buildreqs # command to use to find out if a buildreq is already satisfied debug ("Package buildrequires before filtering: %s" % pkgs[name]['buildreqs']) chkcmd = "rpm --root=%s -q --whatprovides '%%s'" % self.rootdir candidates = pkgs[name]['buildreqs'] buildreqs = [] for cand in candidates: if cand == "kernel": print "Warning: make sure you have manually installed the right kernel rpm" print " and that you are building with --keep" continue elif cand == "kernel-source": print "Warning: make sure you have manually installed the right kernel-source rpm" print " and that you are building with --keep" continue #FIXME: if an older version of a buildrequire is already #installed, it will get filtered even though we need the #newer version #elif commands.getstatusoutput(chkcmd % cand)[0]: buildreqs.append (cand) debug ("BuildRequires after filtering: %s" % buildreqs) # only try to install if filtering left buildreqs if (buildreqs): srcs = get_sources_list (self.config) # this ensures that locally built RPMS are used root = get_config_dir (config, 'root') srcs.insert (0, 'rpm-dir file://%s/usr/src rpm mach-local' % root) create_sources_list (config, srcs) try: self.installer_get (["update", ]) except Root.Error: raise self.Error, 'could not update' try: # wrap all buildreqs in quotes self.installer_get ([ 'install %s' % "'" + "' '".join (buildreqs) + "'", ], "Installing BuildRequires", self.progress ()) except Root.Error: raise self.Error, 'could not install buildreqs %s' % buildreqs #continue else: debug ("No BuildRequires for %s." % srpmname) # get the name and create the place where to store results h = pkgs[name]['header'] name = h[rpm.RPMTAG_NAME] version = h[rpm.RPMTAG_VERSION] release = h[rpm.RPMTAG_RELEASE] # if the release tag needs mangling, do it here as well if self.config['release']: debug ("Mangling release tag with %s" % self.config['release']) fullname = '%s-%s-%s.%s' % (name, version, release, self.config['release']) else: debug ("Not mangling release tag") fullname = '%s-%s-%s' % (name, version, release) resultdir = os.path.join (self.resultdir, fullname) debug ("Storing result in %s" % resultdir) ensure_dir (resultdir) # rebuild binary rpm from the src rpm # install the src.rpm self.do_chroot ("%s -c 'rpm -Uhv /tmp/%s' %s" % (self.config['runuser'], srpmname, builduser)) specfile = pkgs[name]['spec'] command = 'cp /usr/src/rpm/SPECS/%s /tmp/%s' % (specfile, specfile) self.do_chroot(command) command = 'chmod a+r /tmp/%s' % specfile self.do_chroot(command) # copy specfile to resultdir shutil.copy2 (os.path.join (self.rootdir, 'tmp', specfile), resultdir) # mangle spec file if requested if self.config['release']: # first check if it doesn't already comply ! # FIXME: since fedora.us now uses .1 as a disttag, # this checking breaks, so we disable it for now #debug ("self.rootdir: %s" % self.rootdir) try: spec = Spec (os.path.join (self.rootdir, 'tmp', specfile), join_quoted (options)) except Spec.Error, message: raise self.Error, message release = spec.tag ("release") debug ("Release of spec file to mangle is %s" % release) #if not release.endswith ('.' + self.config['release']): if 1: # we quote sed with ' because we quote shell -c with "" # we use [:space:] and ^[:space:], most portable # don't use \s, doesn't work correctly on RH9 command = "sed -e 's,^\\\\(Release[[:space:]]*:[[:space:]]*[^[:space:]]*\\\\)[[:space:]]*$,\\\\1.%s,i' /tmp/%s > /usr/src/rpm/SPECS/%s" % (self.config['release'], specfile, specfile) debug ("mangling with %s" % command) self.do_chroot (command) command = "grep Release: /usr/src/rpm/SPECS/%s" % specfile (ret, output) = self.do_chroot (command) debug ("Mangled to: %s" % output) else: debug ("Release already contains mangle trailer %s" % self.config['release']) # delete the original temp spec file command = 'rm /tmp/%s' % specfile self.do_chroot (command) # detect amount of CPU's on system so _smp_flags gets set # correctly on chroot if not os.environ.has_key('RPM_BUILD_NCPUS'): nrcpus = os.popen('/usr/bin/getconf _NPROCESSORS_ONLN').read().rstrip() else: nrcpus = os.environ['RPM_BUILD_NCPUS'] # run an rpmbuild test using the root argument to check for # fulfilled BuildRequires, using the host's rpmbuild # so that it understands the created database # also make the topdir point to the root's rpm tree, so that # it picks up on Sources: and Patches: copied there command = "rpmbuild -bp --root %s %s " \ "--define '_topdir %s' " \ "--define '_tmppath %s' " \ "--define '__spec_prep_pre exit 0' " \ "--define 'setup :' %s" \ % (self.rootdir, join_quoted (options), os.path.join (self.rootdir, 'usr', 'src', 'rpm'), os.path.join ('/var', 'tmp'), os.path.join (self.rootdir, 'usr', 'src', 'rpm', 'SPECS', specfile)) debug ("Checking buildreqs: %s" % command) try: (status, output) = self.do (command, False) except self.ReturnValue, (retval, output): raise self.Error, "BuildRequires not met:\n%s" % output # rebuild from spec inside chroot, note: using a login shell # in order to get a vanilla default environment # we use --nodeps here to that the build doesn't try to read # the target's database, which might be written in a different # version than the host's rpmbuild understands. # this is fine now since we check for met buildrequires before. command = "%s -c \"RPM_BUILD_NCPUS=%s %s --nodeps -ba %s /usr/src/rpm/SPECS/%s 2>&1\" - %s" % ( self.config['runuser'], nrcpus, self.config['root_rpmbuild'], join_quoted (options), specfile, builduser) (status, output) = self.do_chroot (command, "Rebuilding source rpm %s" % srpmname, True) if (output): open (resultdir + '/rpm.log', "wb"). write (output) if status != 0: sys.stderr.write ("ERROR: something went wrong rebuilding the .src.rpm\n") sys.stderr.write ("ERROR: inspect rpm build log %s/rpm.log\n" % resultdir) self.umount () self.unlock () raise self.Error, "failed to rebuild SRPMs" # reinstall and repackage src rpm #sys.stdout.write ("Repackaging %s ..." % srpm) #do_chroot (config, '%s -Uhv /tmp/%s > /dev/null 2>&1' % (chroot_rpm, srpm)) #do_chroot (config, "%s -c 'rpm -Uhv /tmp/%s > /dev/null 2>&1' %s" % (self.config['runuser'], srpm, builduser)) #self.rpm ('-Uhv /tmp/%s > /dev/null 2>&1' % srpm) #(status, output) = do_chroot_with_output (config, # "%s -c '%s -bs --nodeps /usr/src/rpm/SPECS/%s 2>&1' %s" % (self.config['runuser'], self.config['root_rpmbuild'], spec, builduser), # True) # analyze log file and move all of the rpms listed as Wrote: (srpm, rpms) = get_rpms_from_log (resultdir + '/rpm.log') # FIXME: install these RPMS based on a boolean ? # FIXME: error checks hostrpms = map (lambda x: self.rootdir + x, rpms) output = self.rpm (['-qp', '--qf', "%%{name}=%%{epoch}:%%{version}-%%{release} "] + hostrpms) #(status, output) = do_chroot_with_output (config, '%s -qp --qf "%%{name}=%%{epoch}:%%{version}-%%{release} " %s' % (chroot_rpm, string.join (rpms, " "))) # FIXME: don't install built packages automatically, they can # cause conflicts and break sequence builds # Need to handle packages without Epochs :/ # output = string.replace (output, '=(none):', '='); #sys.stdout.write ("Installing built RPMS and dependencies ...") print "Collecting results of build %s" % srpm rpms.append (srpm) for file in rpms: path = self.rootdir + file # file already contains starting / shutil.copy2 (path, resultdir) # for now, we do, for multiple builds # self.do_chroot ('rm ' + file) # update internal apt/yum repo srcs = get_sources_list (self.config) # this ensures that locally built RPMS are already used for apt-get root = get_config_dir (self.config, 'root') srcs.insert (0, 'rpm-dir file://%s/usr/src rpm mach-local' % root) create_sources_list (self.config, srcs) self.installer_get (["update", ]) print "Build of %s succeeded, results in\n%s" % (fullname, resultdir) resultdirs.append (resultdir) # md5sum packages and spec file for resultdir in resultdirs: cmd = 'cd %s && md5sum *.rpm *.spec > md5sum' % resultdir debug ("running %s" % cmd) os.system (cmd) # now sign packages if requested if self.config['sign']: print "Signing built packages ..." cmd = 'rpm --addsign %s' % string.join (resultdirs, '/*.rpm ') + '/*.rpm' debug ("running %s" % cmd) os.system (cmd) #FIXME: sadly, we cannot clearsign multiple files ? print "clearsigning md5sums ..." for resultdir in resultdirs: if os.path.exists (os.path.join (resultdir, 'md5sum.asc')): os.unlink (os.path.join (resultdir, 'md5sum.asc')) cmd = 'cd %s && md5sum *.rpm > md5sum' % resultdir debug ("running %s" % cmd) os.system (cmd) # FIXME: figure out the proper way to use an agent here cmd = 'gpg --use-agent --clearsign %s/md5sum' % resultdir debug ("running %s" % cmd) os.system (cmd) # run success-script if requested if self.config['script-success'] and self.config['scripts']: for resultdir in resultdirs: cmd = '%s %s' % (self.config['script-success'], resultdir) debug ("running %s" % cmd) os.system (cmd) # now collect stuff to . if requested if self.config['collect']: for resultdir in resultdirs: for x in glob.glob (os.path.join (resultdir, '*.rpm')): os.system ('mv %s %s' % (x, os.path.basename (x))) for x in glob.glob (os.path.join (resultdir, '*')): os.system ('mv %s %s.%s' % (x, os.path.basename (resultdir), os.path.basename (x))) print "Build done." self.unlock () return True # sign all packages at once so we only need the passphrase once # FIXME: repeat if fail # collect all packages if requested self.umount () self.unlock () return def status (self, args): if not os.path.exists (get_config_dir (self.config, 'state')): print "+ %s is cleared" % self.root return lockedness = "unlocked" if self.get_state ('lock'): lockedness = "unlocked" print "+ %s: %s" % (self.root, lockedness) # output of du -B M is inconsistent command = "%s %s du / --max-depth=0 | tail -n 1 | cut -f 1" % (config['chroot'], self.rootdir) try: diskusage = "%d MB" % (string.atoi (string.strip (commands.getoutput (command))) / 1024.0) except ValueError: # atoi failed diskusage = 'Unknown' print " - %s: %s" % (self.rootdir, diskusage) command = "du %s --max-depth=0 | tail -n 1 | cut -f 1" % (self.resultdir) try: diskusage = "%d KB" % string.atoi (string.strip (commands.getoutput (command))) except ValueError: # atoi failed diskusage = 'Unknown' print " - %s: %s" % (self.resultdir, diskusage) if not os.path.exists (self.resultdir): print " - no packages built" return for dir in os.listdir (self.resultdir): # check if there's a .src.rpm built = False for file in os.listdir (os.path.join (self.resultdir, dir)): if file[-8:] == '.src.rpm': built = True if not built: print " - %s: build failed" % dir continue # get disk usage command = "du --max-depth=0 -c %s | tail -n 1 | cut -f 1" % (os.path.join (self.resultdir, dir, '*.rpm')) diskusage = string.strip (commands.getoutput (command)) # get signature command = "rpm -qip %s | grep Signature" % os.path.join (self.resultdir, dir, '*.src.rpm') output = string.strip (commands.getoutput (command)) # get last word of output signature = output.split ()[-1] signedness = None if signature == "(none)": signedness = "unsigned" else: signedness = "signed with %s" % signature print " - %s: %s K, %s " % (dir, diskusage, signedness) print # private methods def _call_setup_hook (self, state): key = 'setup-%s' % state if self.hooks.has_key (key): status = os.system ("%s '%s' '%s' '%s' '%s'" % (self.hooks[key], state, self.root, self.rootdir, self.statedir)) if not os.WIFEXITED (status) or os.WEXITSTATUS (status) != 0: raise Root.Error, "setup-%s-hook failed" % (state) def _setup_prep (self): "prepares a given root if not done yet; this ensures that a number" "of necessary files exist in the root" if self.get_state ('prep'): debug ("target prep already set up") return print "Preparing root" ensure_dir (os.path.join(self.rootdir, 'var', 'lib', 'rpm')) ensure_dir (os.path.join(self.rootdir, 'var', 'log')) #FIXME: os.chmod (root + '/var/lib/rpm', g+s) ensure_dir (os.path.join(self.rootdir, 'dev')) print "Making /dev devices" # we need stuff devices = [('null', 'c', '1', '3', '666'), ('urandom', 'c', '1', '9', '644'), ('random', 'c', '1', '9', '644'), ('full', 'c', '1', '7', '666'), ('ptmx', 'c', '5', '2', '666'), ('tty', 'c', '5', '0', '666'), ('zero', 'c', '1', '5', '666')] try: for (dev, devtype, major, minor, perm) in devices: devpath = os.path.join(self.rootdir, 'dev', dev) cmd = '%s %s -m %s %s %s %s' % (config['mknod'], devpath, perm, devtype, major, minor) if not os.path.exists(devpath): self.do(cmd, progress=False) except Root.ReturnValue, (retval, output): raise Root.Error, "mach-helper could not mknod error was: %s" % output # link fd to ../proc/self/fd devpath = os.path.join(self.rootdir, 'dev', 'fd') os.symlink('../proc/self/fd', devpath) ensure_dir (os.path.join (self.rootdir, 'etc', 'rpm')) open (os.path.join (self.rootdir, 'etc', 'mtab'), 'w') open (os.path.join (self.rootdir, 'etc', 'fstab'), 'w') open (os.path.join (self.rootdir, 'var', 'log', 'yum.log'), 'w') ensure_dir (os.path.join (self.rootdir, 'tmp')) ensure_dir (os.path.join (self.rootdir, 'var', 'tmp')) root = self.rootdir state = self.statedir if (config['installer'] == "yum"): ensure_dir (os.path.join (self.statedir, 'yum/yum.repos.d')) ensure_dir (os.path.join (self.rootdir, 'var/cache/mach')) conf = ''' [main] assumeyes = 1 cachedir = ''' + os.path.join ('/', 'var', 'cache', 'mach') + ''' reposdir = /../../../../..''' + os.path.join (state, 'yum', 'yum.repos.d') self.set_state ("yum.conf", conf) srcs = get_sources_list (config) create_sources_list (config, srcs) self._call_setup_hook ("prep") # write state file self.set_state ("prep") def _setup_minimal (self): # FIXME: right now we only create these once; shouldn't we redo this # every time or something ? debug ("Creating config files") for filename in self.config['files'].keys (): self.config_recreate (filename) self.mount () self._groupinstall ('minimal') self.umount () def _setup_base (self): self.mount () self._groupinstall ('base') self.umount () # ensure if su is installed if not os.path.exists (os.path.join (self.rootdir, self.config['runuser'])): raise self.Error ("su/runuser did not get installed properly") def _check_revert_build (self): # see if we need to revert to the build package list statefile = os.path.join (self.statedir, 'build') self.check_package_list (statefile) def _write_macros(self, file): # write macros from configuration to given file for mac in self.config.get ('macros', {}).items (): file.write ('%%%-19s %s\n' % (mac)) def _setup_build (self): "set up a root for rpm building" # first do the stuff that we want to ensure gets done very time, # for example when config file has changed # create rpm macros debug ("writing macros file") macros = open (self.rootdir + '/tmp/macros', 'w') macros.write ("%_topdir /usr/src/rpm\n") macros.write ("%_rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm\n") macros.write ("%_unpackaged_files_terminate_build 1\n") macros.write ("%_missing_doc_files_terminate_build 1\n") self._write_macros (macros) macros.close () self.do_chroot ('mv /tmp/macros /etc/rpm', fatal = True) # now check if we're already good if self.get_state ('build'): self._check_revert_build () return self.mount () # install packages self._groupinstall ('build') #FIXME: add mach user and group self.do_chroot ("echo %s:x:500:500::/usr/src/rpm:/bin/bash >> /etc/passwd" % builduser, fatal = True) self.do_chroot ("echo %s::500:%s >> /etc/group" % (buildgroup, builduser), fatal = True) self.do_chroot ("mkdir -p /usr/src/rpm", fatal = True) self.do_chroot ("chown -R %s:%s /usr/src/rpm" % (builduser, buildgroup), fatal = True) self.do_chroot ("%s -c 'cp -p /etc/skel/.bashrc /usr/src/rpm/.bashrc || cp -p /etc/bashrc /usr/src/rpm/.bashrc || :' %s" % ( self.config['runuser'], builduser), fatal = True) # create /boot/kernel.h with a warning self.do_chroot ("mkdir -p /boot", fatal = True) self.do_chroot ("echo '#ifndef __BOOT_KERNEL_H_' > /boot/kernel.h", fatal = True) self.do_chroot ("echo '#define __BOOT_KERNEL_H_' >> /boot/kernel.h", fatal = True) self.do_chroot ("echo '#error This is a kernel.h generated by mach, including this indicates a build error !' >> /boot/kernel.h", fatal = True) self.do_chroot ("echo '#endif /* __BOOT_KERNEL_H_ */' >> /boot/kernel.h", fatal = True) # create dir structure for dir in ('RPMS', 'SRPMS', 'SOURCES', 'SPECS', 'BUILD'): self.do_chroot ("%s -c 'mkdir -p /usr/src/rpm/%s' %s" % (self.config['runuser'], dir, builduser), fatal = True) # this ensures that locally built RPMS are already used for apt-get for dir in ('RPMS', 'SRPMS'): self.do_chroot ('ln -sf %s /usr/src/rpm/%s.mach-local' % (dir, dir), fatal = True) self.installer_get (["update", ]) self._call_setup_hook ("build") self.stdout ("Making snapshot ...\n") # FIXME: use _with_output for this output = self.rpm (['-qa', '--qf', qfmt, ]) self.set_state ('build', output) self.umount () def _install (self, set): "installs the given group into the root." "Packages in this set are defined in dist." debug ("_install (%s)" % set) if self.get_state (set): return debug ("Installing set: %s" % set) try: rpms = self.config['packages'][set] except KeyError: raise self.Error, "Set '%s' not defined for this distribution !" % set if rpms: try: self.installer_get(['install %s' % rpms, ], "Installing package set '%s'" % set, True) except self.Error: self.installer_get (['install %s' % rpms, ], "Retrying installing package set '%s'" % set, True) else: print "No packages in %s set" % rpms self.set_state (set) def _groupinstall (self, set): "installs the given set of packages into the root." "Packages in this set are defined in dist." debug ("_groupinstall (%s)" % set) if self.get_state (set): return debug ("Installing groups for %s" % set) try: groups = self.config['groups'][set] except KeyError: raise self.Error, "Group set '%s' not defined for this distribution!" % set if groups: self.group_installer(groups, "Installing group '%s'" % set, True) else: print "No groups defined for %s set" % set self.set_state (set) ### helper functions # return an argument string containing all of the arguments of the given list, # but each of them surrounded by quotes # this is done so that ['define', 'kernel 2.4.22'] returns a string like # "define" "kernel 2.4.22" # so that using mach build --define "kernel 2.4.22" passes correct arguments # to rpmbuild def join_quoted (args): if not args: return "" else: return "'" + string.join (args, "' '") + "'" # return path to rpm binary suited to our target root, extracting it if # necessary # FIXME: this function is not used right now, would it be a good idea to # do so in the future ? def get_rpm (config): debug ("getting rpm for root %s" % config['root']) tmpdir = get_config_dir (config, 'tmp') binary = os.path.join (tmpdir, 'bin', 'rpm') # first check if it exists yet if os.path.exists (binary): return binary # it doesn't, so get it packagesdir = get_packages_dir (config) # find rpm package in 'base' set packages = config['packages']['base'] matchstr = re.compile ("^rpm-\d") rpm = "" for package in packages: if matchstr.search (package): rpm = package if not rpm: sys.stderr.write ("Could not find rpm in base set !\n") exit (config) print "Found rpm %s" % rpm # check if it exists path = os.path.join (packagesdir, rpm) if not os.path.exists (path): sys.stderr.write ("Could not find file %s !\n" % path) exit (config) # extract rpm binary from it ensure_dir (tmpdir) command = "cd %s && rpm2cpio %s | cpio -id --no-absolute-filenames bin/rpm > /dev/null" % (tmpdir, path) if os.system (command) != 0: sys.stderr.write ("Could not extract rpm binary from package !\n") exit (config) return binary # grab given url and store it as filename, preserving timestamps def urlgrab (url, filename): print "Getting %s ..." % url opener = urllib.URLopener () try: (t, h) = opener.retrieve (url, filename) except IOError, e: sys.stderr.write ("Couldn't retrieve %s:\n%s\n" % (url, e)) raise e d = None try: if h: d = time.mktime (h.getdate ("Last-Modified") or h.getdate ("Date")) if d: os.utime (filename, (d, d)) except: sys.stderr.write ("Warning: time stamp not preserved for %s\n" % filename) if config.get ('hooks', {}).has_key ('download'): status = os.system ("%s '%s' '%s'" % (config['hooks']['download'], url, filename)) if not os.WIFEXITED (status) or os.WEXITSTATUS (status) != 0: raise Root.Error, "Download-hook for %s failed" % url # unlock and exit def exit (config): unlock (config) sys.exit (1) # run a command, opening a pty and returning fd and pid def pty_popen (cmd): pid, fd = os.forkpty () if pid == 0: os.execl ('/bin/sh', 'sh', '-c', cmd) else: return os.fdopen (fd), pid # run a command and give a progress spinner that regularly advances # delta gives the number of secs between each spin iteration # jump is the number of iterations before jumping one dot further # returns a tuple: # - the return status of the app # - all of the output as one big string def do_progress (cmd, delta = 0.1, jump = 20): # choose a random spinner by jumping through lots of hoops spinkey = random.randint (0, len (config['spinner'].keys ()) - 1) spinner = config['spinner'][config['spinner'].keys ()[spinkey]] i = 0 # jump counter j = 0 # spinner state timestamp = time.time () running = 1 size = 0 # size of total output since last spinner update output = [] (fdin, pid) = pty_popen (cmd) fcntl.fcntl (fdin, fcntl.F_SETFL, os.O_NONBLOCK) # don't block input sys.stdout.write (spinner[0]) sys.stdout.flush () # otherwise we don't see nothing until first update while running: block = "" try: block = fdin.read () # trash all \r\n block = string.replace (block, '\r\n', '\n') output.append (block) except IOError: pass size = size + len (block) if size > 0 and time.time () > timestamp + delta: timestamp = time.time () size = 0 i = i + 1 j = (j + 1) % len (spinner) sys.stdout.write ('\b' + spinner[j % len (spinner)]) if (i > jump): i = 0 sys.stdout.write ('\b.%s' % spinner[j]) sys.stdout.flush () time.sleep (0.1) try: (dip, status) = os.waitpid (pid, os.WNOHANG) except OSError: running = 0 # done output = string.join (output, '').split ("\n") retval = 0 if os.WIFEXITED (status): retval = os.WEXITSTATUS (status) if (retval != 0): sys.stdout.write ('\b!\n') sys.stderr.write ('error: %s failed.\n %s\n' % (cmd, string.join (output, "\n"))) else: sys.stdout.write ('\b.\n') return (retval, string.join (output, '\n')) # execute the given command; possibly with progress bar # return (return value, output) of command def do_with_output (command, progress = False): debug ("Executing %s" % command) if progress: return do_progress (command) else: (status, output) = commands.getstatusoutput (command) if os.WIFEXITED (status): retval = os.WEXITSTATUS (status) #if (retval != 0): # sys.stdout.write ('\b!\n') #sys.stderr.write ('error:\n %s' % string.join (output)) #else: # sys.stdout.write ('\b.\n') #FIXME: hack to return no output, we should change this if (retval != 0): sys.stderr.write ("error: %s failed\n" % command) return (retval, output) # do a topological sort on the given list of pairs # taken from a mail on the internet and adapted # given the list of pairs of X needed for Y, return a suggested build list class CycleError(Exception): pass def topological_sort (pairlist): numpreds = {} # elt ->; # of predecessors successors = {} # elt -> list of successors for first, second in pairlist: # make sure every elt is a key in numpreds if not numpreds.has_key (first): numpreds[first] = 0 if not numpreds.has_key (second): numpreds[second] = 0 # since first < second, second gains a pred ... numpreds[second] = numpreds[second] + 1 # ... and first gains a succ if successors.has_key (first): successors[first].append (second) else: successors[first] = [second] # suck up everything without a predecessor answer = filter (lambda x, numpreds=numpreds: numpreds[x] == 0, numpreds.keys()) # for everything in answer, knock down the pred count on # its successors; note that answer grows *in* the loop for x in answer: del numpreds[x] if successors.has_key (x): for y in successors[x]: numpreds[y] = numpreds[y] - 1 if numpreds[y] == 0: answer.append (y) # following "del" isn't needed; just makes # CycleError details easier to grasp del successors[x] if numpreds: # everything in numpreds has at least one successor -> # there's a cycle raise CycleError, (answer, numpreds, successors) return answer # get a header from a path to a local src.rpm; or False if no header def get_header (path): debug ("Getting RPM header from %s" % path) # ignore signatures ts = rpm.TransactionSet ("", (rpm._RPMVSF_NOSIGNATURES)) fd = os.open (path, os.O_RDONLY) try: h = ts.hdrFromFdno (fd) except: print "a whoopsie occurred." if not h: sys.stderr.write ("ERROR: %s doesn't look like a .src.rpm\n" % path) return False return h # get all of the written rpms from the log file and return a tuple # srpm, list of rpms def get_rpms_from_log (log): rpms = [] srpm = '' matchstr = re.compile ("src\.rpm$") output = commands.getoutput ('grep "Wrote: " %s | sort | uniq' % log) for line in output.split ("\n"): curr = string.replace (line, 'Wrote: ', '') if matchstr.search (curr): srpm = curr else: rpms.append (curr) return (srpm, rpms) # print out debug info if wanted # uses DEBUG global def debug (string): if DEBUG: print "DEBUG: %s" % string # return full path to directory based on the root (in config) and which sort # of directory to get def get_config_dir (config, which): #FIXME: do error checking on key return config['dirs'][which + 's'] + '/' + config['root'] # return full path to directory holding bootstrap packages for this root # DELETEME def get_packages_dir (config): return config['dirs']['packages'] + '/' + config['packages']['dir'] # make sure a dir exists def ensure_dir (dir): debug ("ensuring dir %s" % dir) if not os.path.exists (dir): try: os.makedirs (dir) except OSError: sys.stderr.write ("Could not create %s, make sure you have permissions to do so\n" % dir) sys.exit (1) # get the list of sources.list lines based on the config # based on sourceslist dict def get_sources_list (config): sourceslist = [] root = config['root'] for platform in config['sourceslist'][root].keys (): for source in config['sourceslist'][root][platform]: try: if (config['installer'] == "yum"): sourceslist.append (config['yumsources'][platform][source]) except KeyError: if (config['installer'] == "yum"): sys.stderr.write ("no %s key found in config['yumsources'][%s]\n" % (source, platform)) sys.exit (1) debug ("sources.list: " + string.join (sourceslist, "\n")) return sourceslist # create sources.list file by putting the given list of lines in it def create_sources_list (config, list): root = get_config_dir (config, 'root') statedir = get_config_dir (config, 'state') if (config['installer'] == "yum"): sources = open (statedir + '/yum/yum.repos.d/yum.repo', 'w') for source in list: fields = source.split () sources.write("[%s]\nname=%s\nbaseurl=%s/%s\nenabled=1\ngpgcheck=0\n" % (fields[3], fields[3], fields[1], fields[2])) ensure_dir (os.path.join (root, 'var/cache/mach/%s/packages' % fields[3])) ensure_dir (os.path.join (root, 'var/cache/mach/%s/headers' % fields[3])) if (fields[0] == 'rpm-dir'): rootobj = Root (config) rootobj.do_chroot ('createrepo -x S*/* -x B*/* /usr/src/rpm') sources.close () # lock a given root; creates a lock file in the statedir under root def lock (config): statedir = get_config_dir (config, 'state') ensure_dir (statedir) lockpath = statedir + '/lock' if os.path.exists (lockpath): if not config['force']: sys.stderr.write ('error: %s already locked !\n' % config['root']) return False else: print 'warning: overriding lock on root %s' % config['root'] try: lockfile = open (lockpath, 'w') except: sys.stderr.write ('error: can''t create lock file for %s !\n' % config['root']) return False lockfile.close () return True # unlock a given root; removes lock file in the statedir under root def unlock (config): statedir = get_config_dir (config, 'state') lockpath = statedir + '/lock' if os.path.exists (lockpath): os.remove (lockpath) return True # return the state, or nothing if state isn't there def get_state (config, state): statedir = get_config_dir (config, 'state') if os.path.exists (statedir + '/' + state): return statedir + '/' + state return # check if everything is in order to actually do stuff def sanity_check (config): # check if we're in the mach group if os.getuid () != 0: machgid=grp.getgrnam ('mach')[2] if not machgid in os.getgroups(): sys.stderr.write ("error: user is not in group mach, please add !\n") sys.exit (1) # check if partial dir exists partial = config['dirs']['archives'] + '/partial' ensure_dir (partial) # check if mach-helper is suid if not (os.stat ('/usr/local/sbin/mach-helper')[0] & stat.S_ISUID): sys.stderr.write ("error: /usr/local/sbin/mach-helper is not setuid !\n") sys.exit (1) # we're fine return True # print out status for each of the roots set up # FIXME: maybe also add result dirs, since we could have results for cleaned # roots, or no results for roots that are set up def status (config): statedir = config['dirs']['states'] dirs = os.listdir (statedir) for dir in dirs: # only give status if it's a known root dir config['root'] = dir root = Root (config) root.status (None) # main function def main (config, args): global DEBUG # we might change it canonify = 0 cli_config = {} try: opts, args = getopt.getopt (args, 'r:hdfkscqv', ['root=', 'help', 'debug', 'force', 'keep', 'sign', 'collect', 'quiet', 'release=', 'no-scripts', 'canonify', 'version']) except getopt.error, exc: sys.stderr.write ('error: %s\n' % str (exc)) sys.exit (1) # parse environment try: root = os.environ['MACH_ROOT'] except: root = config['defaultroot'] # parse config options for opt, arg in opts: if opt in ('-h', '--help'): print usage print help sys.exit (0) if opt in ('-v', '--version'): print "mach (make a chroot) 0.4.6.1" print "Written by Thomas Vander Stichele ." sys.exit (0) elif opt in ('-r', '--root'): root = arg elif opt in ('-d', '--debug'): DEBUG = 1 elif opt in ('-f', '--force'): cli_config['force'] = 1 elif opt in ('-k', '--keep'): cli_config['keep'] = 1 elif opt in ('-s', '--sign'): cli_config['sign'] = 1 elif opt in ('-c', '--collect'): cli_config['collect'] = 1 elif opt in ('-q', '--quiet'): cli_config['quiet'] = 1 elif opt == '--release': cli_config['release'] = arg elif opt == '--no-scripts': cli_config['scripts'] = 0 elif opt == '--canonify': canonify = 1 # resolve root aliases to real root names, check for duplicates seen_aliases = [] for name in aliases.keys (): if name in seen_aliases: sys.stderr.write ('Root "%s" is ambiguous\n' % al) sys.exit (1) seen_aliases.append (name) for al in aliases[name]: if al in seen_aliases: sys.stderr.write ('Root "%s" is ambiguous\n' % al) sys.exit (1) if al == root: root = name seen_aliases.append (al) del seen_aliases if canonify: print root sys.exit(0) debug ("This is mach (make a chroot) 0.4.6.1") debug ("real root name is %s" % root) # pull in root-specific configuration # FIXME: even nicer would be if we could just do config[rootname] = ... # and copy that over try: config['packages'] = packages[root] except KeyError: sys.stderr.write ('No definition for packages found for %s\n' % root) sys.exit (1) try: config['yumsources'] = yumsources config['sourceslist'] = sourceslist except KeyError: sys.stderr.write ('No sources information found for %s\n' % root) sys.exit (1) try: config['groups'] = groups[root] except KeyError: sys.stderr.write ('No definition for groups found for %s\n' % root) sys.exit (1) if config.has_key (root): for key in config[root].keys (): debug ("setting config['%s'] to %s" % (key, config[root][key])) config[key] = config[root][key] for key in cli_config.keys(): debug ("setting config['%s'] to %s from CLI" % (key, cli_config[key])) config[key] = cli_config[key] # debug output options debug ("root: %s" % root) # process options config['root'] = root # check if everything is ready to go sanity_check (config) # run command if not args: print usage print help sys.exit (1) debug ("main: args: %s" % args) command = args[0] debug ("main: running %s" % command) root = Root (config) output = "" if command in allowed_commands: # no '-' allowed is silly but what can we do about it ? # this is also a good place to intercept commands that need to # be interactive if command == "status": status (config) return if not command in Root.__dict__.keys(): sys.stderr.write ("No %s method defined\n" % command) sys.exit (1) try: # we changed behaviour to pass the list of args instead of a string output = Root.__dict__[command] (root, args[1:]) except Root.Locked: sys.stderr.write ("Root is locked. Use -f to override lock.\n") except Root.Error, message: sys.stderr.write ("ERROR: %s\n" % message) root.unlock () sys.exit (1) except Root.ReturnValue, (retval, output): sys.stderr.write ("Return value: %s\n" % retval) root.unlock () sys.exit (retval) except KeyboardInterrupt: sys.stderr.write ("Aborted.\n") root.unlock () sys.exit (1) if output and output != True: print output else: sys.stderr.write ("No such command '%s'\n" % command) # run main program if __name__ == '__main__': # run command specified or go into interpreter mode if len (sys.argv) > 1: main (config, sys.argv[1:]) else: print "starting mach interpreter ..." running = True; while running: sys.stdout.write ("> ") command = sys.stdin.readline () main (config, string.split (command))