# This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Library General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # Copyright 2005 Dan Williams and Red Hat, Inc. import threading import sys import os import shutil import string import fcntl import urllib import errno import exceptions import time import Builder import Config from plague import ExecUtils from plague import FileDownloader from plague import FileTransfer class BuilderMock(threading.Thread): """puts things together for an arch - baseclass for handling builds for other arches""" def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): self._controller = controller self._buildarch = buildarch self._jobname = jobname self._starttime = time.time() self._endtime = 0 self._mockstarttime = 0 self._uniqid = uniqid self._status = 'init' self._die = False self._repo_locked = True self._repo_wait_start = 0 self._files = {} self._childpid = 0 self._target_cfg = target_cfg self._builder_cfg = target_cfg.parent_cfg() self._srpm_path = None self._srpm_wait_start = 0 self._log_fd = None self._mock_config = None self._done_status = '' self._mock_log = None self._buildroot = self._target_cfg.mock_config() self._downloader = None self._uploader = None self.arch_command = "" self._work_dir = os.path.abspath(self._builder_cfg.get_str("Directories", "builder_work_dir")) self._source_dir = os.path.join(self._work_dir, self._uniqid, "source") if not os.path.exists(self._source_dir): os.makedirs(self._source_dir) self._result_dir = os.path.join(self._work_dir, self._uniqid, "result") if not os.path.exists(self._result_dir): os.makedirs(self._result_dir) self._state_dir = os.path.join(self._work_dir, self._uniqid, "mock-state") if not os.path.exists(self._state_dir): os.makedirs(self._state_dir) logfile = os.path.join(self._result_dir, "job.log") self._log_fd = open(logfile, "w+") threading.Thread.__init__(self) def starttime(self): return self._starttime def endtime(self): return self._endtime def die(self): if self.is_done_status() or self._done_status == 'killed': return True self._die = True return True def _handle_death(self): self._die = False self._done_status = 'killed' self._log("Killing build process...") if self._downloader: self._downloader.cancel() if self._uploader: self._uploader.cancel() # Don't try to kill a running cleanup process if self._status != 'cleanup': # Kill a running non-cleanup mock process, if any if self._childpid: child_pgroup = 0 - self._childpid try: # Kill all members of the child's process group os.kill(child_pgroup, 9) except OSError, exc: self._log("ERROR: Couldn't kill child process group %d: %s" % (child_pgroup, exc)) else: # Ensure child process is reaped self._log("Waiting for mock process %d to exit..." % self._childpid) try: (pid, status) = os.waitpid(self._childpid, 0) except OSError: pass self._log("Mock process %d exited." % self._childpid) self._childpid = 0 # Start cleanup up the job self._start_cleanup() self._log("Killed."); def _log(self, msg, newline=True): if msg and self._log_fd: if newline: msg = msg + "\n" self._log_fd.write(msg) self._log_fd.flush() os.fsync(self._log_fd.fileno()) if self._builder_cfg.get_bool("General", "debug"): logtext = "%s: %s" % (self._uniqid, msg) sys.stdout.write(logtext) sys.stdout.flush() def _copy_mock_output_to_log(self): if self._mock_log and os.path.exists(self._mock_log): ml = open(self._mock_log, "r") line = "foo" while len(line): line = ml.readline() if len(line): self._log_fd.write(line) ml.close() os.remove(self._mock_log) self._mock_log = None def _start_build(self): self._log("Starting step 'building' with command:") if not os.path.exists(self._result_dir): os.makedirs(self._result_dir) # Set up build process arguments args = [] builder_cmd = os.path.abspath(self._builder_cfg.get_str("General", "builder_cmd")) cmd = builder_cmd if self.arch_command and len(self.arch_command): arg_list = self.arch_command.split() for arg in arg_list: args.append(arg) cmd = os.path.abspath(arg_list[0]) args.append(builder_cmd) args.append("-r") args.append(self._buildroot) args.append("--arch") args.append(self._buildarch) args.append("--resultdir=%s" % self._result_dir) args.append("--statedir=%s" % self._state_dir) args.append("--uniqueext=%s" % self._uniqid) args.append(self._srpm_path) self._log(" %s" % string.join(args)) self._mock_log = os.path.join(self._result_dir, "mock-output.log") self._childpid = ExecUtils.exec_with_redirect(cmd, args, None, self._mock_log, self._mock_log) self._mockstarttime = time.time() self._status = 'prepping' def _start_cleanup(self): self._log("Cleaning up the buildroot...") args = [] builder_cmd = os.path.abspath(self._builder_cfg.get_str("General", "builder_cmd")) cmd = builder_cmd if self.arch_command and len(self.arch_command): arg_list = self.arch_command.split() for arg in arg_list: args.append(arg) cmd = os.path.abspath(arg_list[0]) args.append(builder_cmd) args.append("clean") args.append("--uniqueext=%s" % self._uniqid) args.append("-r") args.append(self._buildroot) self._log(" %s" % string.join(args)) self._childpid = ExecUtils.exec_with_redirect(cmd, args, None, None, None) self._status = 'cleanup' def _mock_is_prepping(self): mock_status = self._get_mock_status() if mock_status: if mock_status == 'prep': return True elif mock_status == 'setu': return True return False def _mock_using_repo(self): mock_status = self._get_mock_status() if mock_status: if mock_status == 'init': return True elif mock_status == 'clea': return True elif mock_status == 'prep': return True elif mock_status == 'setu': return True return False def _mock_is_closed(self): mock_status = self._get_mock_status() if mock_status and mock_status == "done": return True return False def _get_mock_status(self): mockstatusfile = os.path.join(self._state_dir, 'status') if not os.path.exists(mockstatusfile): return None f = open(mockstatusfile, "r") fcntl.fcntl(f.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) while True: try: f.seek(0, 0) mockstat = f.read(4) except OSError, exc: if exc.errno == errno.EAGAIN: try: time.sleep(0.25) except KeyboardInterrupt: pass continue else: if len(mockstat) < 4: continue break f.close() return mockstat.lower() def _read_mock_config(self): mockconfigfile = os.path.join(self._result_dir, 'mockconfig.log') if not os.path.exists(mockconfigfile): return None f = open(mockconfigfile, "r") contents = {} for line in f: (item, loc) = line.split('=') item = item.strip() loc = loc.strip() contents[item] = loc f.close() return contents def _srpm_download_cb(self, dl_status, cb_data, err_msg=None): # We might have been cancelled if self.is_done_status(): return url = cb_data if dl_status == FileTransfer.FT_RESULT_SUCCESS: self._status = 'downloaded' self._log("Retrieved %s." % url) elif dl_status == FileTransfer.FT_RESULT_FAILED: self._log("ERROR: Failed to retrieve '%s' because: %s" % (url, err_msg)) self._post_cleanup('failed') elif dl_status == FileTransfer.FT_RESULT_CANCELED: # Ignore cancelation pass self._downloader = None def _status_init(self): if not self._srpm_wait_start: self._srpm_wait_start = time.time() # Kill a job that's waiting for its SRPM URL # after 30 minutes because it's likely orphaned if time.time() > (self._srpm_wait_start + (60 * 30)): self._log("Job waited too long waiting for its SRPM URL. Killing it...") self._post_cleanup('failed') def notify_srpm_url(self, srpm_url): """Called by our controlling Builder object to tell us that the server has sent our SRPM URL.""" if self._status != "init": return err_msg = None try: # Validate the SRPM URL and the SRPM's filename srpm_filename = FileDownloader.get_base_filename_from_url(srpm_url, ['.src.rpm']) self._srpm_path = os.path.join(self._source_dir, srpm_filename) success = True except FileDownloader.FileNameException, exc: err_msg = "ERROR: SRPM file name was invalid. Message: '%s'" % exc if not err_msg: # Ask our controlling builder to download it (if an Active builder) # or to move it for us (if a Passive builder) self._status = 'downloading' try: target_dir = os.path.dirname(self._srpm_path) self._downloader = self._controller.download_srpm(self._uniqid, srpm_url, target_dir, self._srpm_download_cb, srpm_url) except FileDownloader.FileNameException, exc: err_msg = "ERROR: Failed to begin SRPM download. Error: '%s' URL: %s" % (exc, srpm_url) if err_msg: self._log(err_msg) self._post_cleanup('failed') def _status_downloading(self): pass def _status_downloaded(self): # We can't start doing anything with yum until the build # server tells us the repo is unlocked. if not self._repo_locked: self._start_build() return # Only show this message once if self._repo_wait_start <= 0: self._log("Waiting for repository to unlock before starting the build...") self._repo_wait_start = time.time() # Kill a job in 'downloaded' state after 30 minutes because # it's likely orphaned if time.time() > (self._repo_wait_start + (60 * 30)): self._log("Job waited too long for repo to unlock. Killing it...") self._post_cleanup('failed') def _watch_mock(self, good_exit, bad_exit): (aux_pid, status) = os.waitpid(self._childpid, os.WNOHANG) status = os.WEXITSTATUS(status) if aux_pid: self._childpid = 0 if status == 0: self._done_status = good_exit elif status > 0: self._done_status = bad_exit self._copy_mock_output_to_log() self._start_cleanup() def _status_prepping(self): # Refresh mock status to see whether it changed during sleep. # Avoid mock/plague race condition. if not self._mock_is_prepping(): if not self._mock_using_repo(): # status changed during sleep self._status = 'building' return # Mock shouldn't exit at all during the prepping stage, if it does # something is wrong self._watch_mock('failed', 'failed') if self._status != 'prepping': return # We need to make sure that mock has dumped the status file withing a certain # amount of time, otherwise we can't tell what it's doing mockstatusfile = os.path.join(self._state_dir, 'status') if not os.path.exists(mockstatusfile): # something is wrong if mock takes more than 15s to write the status file if time.time() > self._mockstarttime + 15: self._mockstarttime = 0 self._log("ERROR: Timed out waiting for the mock status file! %s" % mockstatusfile) self._post_cleanup('failed') else: if not self._mock_config and self._mock_is_prepping(): self._mock_config = self._read_mock_config() def _status_building(self): self._watch_mock('done', 'failed') def _status_cleanup(self): (aux_pid, status) = os.waitpid(self._childpid, os.WNOHANG) if aux_pid: # Mock exited self._childpid = 0 if self._mock_config: if self._mock_config.has_key('rootdir'): mock_root_dir = os.path.abspath(os.path.join(self._mock_config['rootdir'], "../")) # Ensure we're actually deleteing the job's rootdir if mock_root_dir.endswith(self._uniqid): shutil.rmtree(mock_root_dir, ignore_errors=True) if self._mock_config.has_key('statedir'): shutil.rmtree(self._mock_config['statedir'], ignore_errors=True) source_dir = os.path.abspath(os.path.join(self._mock_config['rootdir'], "../source")) # Ensure we're actually deleteing the job's sourcedir if source_dir.endswith(os.path.join(self._uniqid, "source")): shutil.rmtree(source_dir, ignore_errors=True) # Ensure child process is reaped if it's still around if self._childpid: try: self._log("Waiting for child process %d to exit." % self._childpid) (pid, status) = os.waitpid(self._childpid, 0) except OSError: self._childpid = 0 pass if self._done_status is not 'killed': self._copy_mock_output_to_log() self._post_cleanup() def _post_cleanup(self, done_status=None): if done_status: self._done_status = done_status # If we're killed, we don't care about uploading logs to the build server if self._done_status is not 'killed': self._status = "uploading" self._files = self._find_files() self._uploader = self._controller.upload_files(self._uniqid, self._files, self._upload_cb, None) else: self._status = self._done_status def _upload_cb(self, status, cb_data, msg): if status == FileTransfer.FT_RESULT_SUCCESS: pass elif status == FileTransfer.FT_RESULT_FAILED: self._done_status = 'failed' self._log("Job failed because files could not be uploaded: %s" % msg) self._status = self._done_status self._uploader = None def _status_uploading(self): pass def _job_done(self): self._log("-----------------------") if self._status == 'done': self._log("Job completed successfully.") elif self._status == 'failed': self._log("Job failed due to build errors! Please see build logs.") elif self._status == 'killed': self._log("Job failed because it was killed.") self._log("\n") if self._log_fd: self._log_fd.close() self._log_fd = None def run(self): # Print out a nice message at the start of the job target_str = Config.make_target_string_from_dict(self._target_cfg.target_dict()) self._log("Starting job:") self._log(" Name: %s" % self._jobname) self._log(" UID: %s" % self._uniqid) self._log(" Arch: %s" % self._buildarch) self._log(" Time: %s" % time.asctime(time.localtime(self._starttime))) self._log(" Target: %s" % target_str) # Main build job work loop while not self.is_done_status(): if self._die: self._handle_death() # Execute operations for our current status try: func = getattr(self, "_status_%s" % self._status) except AttributeError: self._log("ERROR: internal builder inconsistency, didn't recognize status '%s'." % self._status) self._post_cleanup('failed') else: func() time.sleep(3) self._job_done() self._endtime = time.time() if self._childpid: self._log("ERROR: childpid was !NULL (%d)" % self._childpid) self._controller.notify_job_done(self) def _find_files(self): # Grab the list of files in our job's result dir and URL encode them files_in_dir = os.listdir(self._result_dir) file_list = [] self._log("\n") self._log("Output File List:") self._log("-----------------") log_files = [] rpms = [] # sort into logs first, rpms later for fname in files_in_dir: fpath = os.path.join(self._result_dir, fname) if fpath.endswith(".log"): log_files.append(fpath) else: rpms.append(fpath) # Dump file list to log file_list = log_files + rpms i = 1 num_files = len(file_list) for fpath in file_list: self._log(" File (%d of %d): %s" % (i, num_files, os.path.basename(fpath))) i = i + 1 self._log("-----------------") return file_list def status(self): return self._status def uniqid(self): return self._uniqid def files(self): return self._files def unlock_repo(self): self._repo_locked = False def is_done_status(self): if (self._status is 'done') or (self._status is 'killed') or (self._status is 'failed'): return True return False class InvalidTargetError(exceptions.Exception): pass class i386Arch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) self.arch_command = "/usr/bin/setarch i686" class x86_64Arch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) class PPCArch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) self.arch_command = "/usr/bin/setarch ppc32" class PPC64Arch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) class SparcArch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) self.arch_command = "/usr/bin/sparc32" class Sparc64Arch(BuilderMock): def __init__(self, controller, target_cfg, buildarch, jobname, uniqid): BuilderMock.__init__(self, controller, target_cfg, buildarch, jobname, uniqid) self.arch_command = "/usr/bin/sparc64" BuilderClassDict = { 'i386': i386Arch, 'i486': i386Arch, 'i586': i386Arch, 'i686': i386Arch, 'athlon': i386Arch, 'x86_64': x86_64Arch, 'amd64': x86_64Arch, 'ia32e': x86_64Arch, 'ppc': PPCArch, 'ppc32': PPCArch, 'ppc64': PPC64Arch, 'sparc': SparcArch, 'sparcv8': SparcArch, 'sparcv9': SparcArch, 'sparcv9v': SparcArch, 'sparc64': Sparc64Arch, 'sparc64v': Sparc64Arch }