buildbot: lots of improvements

README.md

@@ -10,7 +10,7 @@ cleanbuild:
     true / false
 timeout:
     30 (30 mins, int only)
-extra: (wip)
+extra:
     - update:
         - /bin/true
     - prebuild:

buildbot.py

@@ -114,7 +114,10 @@ class jobsManager:
         if (REPO_ROOT / pkgdirname).exists() and clean:
             self.reset_dir(pkgdirname)
         updates = updmgr.check_update(rebuild_package=pkgdirname)
-        if updates and len(updates) == 1:
+        if not (REPO_ROOT / pkgdirname).exists():
+            ret = f'rebuild failed: no such dir {pkgdirname}'
+            logger.warning(ret)
+        elif updates and len(updates) == 1:
             (pkgconfig, ver, buildarchs) = updates[0]
             march = True if len(buildarchs) >= 2 else False
             for arch in buildarchs:
@@ -123,7 +126,7 @@ class jobsManager:
             ret = f'rebuild job added for {pkgdirname} {" ".join(buildarchs)}'
             logger.info(ret)
         else:
-            ret = f'rebuild failed: no such dir {pkgdirname}'
+            ret = 'rebuild failed: cannot check update.'
             logger.warning(ret)
         return ret
     def _new_buildjob(self, job):
@@ -143,7 +146,7 @@ class jobsManager:
         self.__buildjobs.append(job)
     def __get_job(self):
         if self.__curr_job:
-            logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
+            logger.error(f'Job {self.__curr_job} failed and is not cleaned.')
             self.__finish_job(self.__curr_job, force=True)
             return self.__get_job()
         jobs = self.__buildjobs
@@ -155,6 +158,12 @@ class jobsManager:
             assert pkgdir == self.__curr_job.pkgconfig.dirname
         self.__curr_job = None
         return True
+    def clean_failed_job(self):
+        if self.__curr_job:
+            logger.error(f'Job {self.__curr_job} failed. Correct the error and rebuild')
+            self.__finish_job(self.__curr_job, force=True)
+        else:
+            raise RuntimeError('Unexpected behavior')
     def __makepkg(self, job):
         cwd = REPO_ROOT / job.pkgconfig.dirname
         if job.multiarch:
@@ -164,10 +173,38 @@ class jobsManager:
             mkcmd = MAKEPKG_MAKE_CMD_CLEAN if job.pkgconfig.cleanbuild \
                                         else MAKEPKG_MAKE_CMD
         logger.info('makepkg in %s %s', job.pkgconfig.dirname, job.arch)
-        return mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
+        # run pre-makepkg-scripts
-                                logfile = cwd / 'buildbot.log.makepkg',
+        logger.info('running pre-build scripts')
-                                short_return = True,
+        for scr in getattr(job.pkgconfig, 'prebuild', list()):
-                                seconds=job.pkgconfig.timeout*60)
+            if type(scr) is str:
+                try:
+                    mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
+                except Exception:
+                    print_exc_plus()
+        # actually makepkg
+        try:
+            ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
+                                    logfile = cwd / 'buildbot.log.makepkg',
+                                    short_return = True,
+                                    seconds=job.pkgconfig.timeout*60)
+        except Exception:
+            logger.error(f'Job {job} failed. Running build-failure scripts')
+            for scr in getattr(job.pkgconfig, 'failure', list()):
+                if type(scr) is str:
+                    try:
+                        mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
+                    except Exception:
+                        print_exc_plus()
+            raise
+        # run post-makepkg-scripts
+        logger.info('running post-build scripts')
+        for scr in getattr(job.pkgconfig, 'postbuild', list()):
+            if type(scr) is str:
+                try:
+                    mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=scr, seconds=60*60)
+                except Exception:
+                    print_exc_plus()
+        return ret
     def __clean(self, job, remove_pkg=False, rm_src=True):
         cwd = REPO_ROOT / job.pkgconfig.dirname
         logger.info('cleaning build dir for %s, %sremoving pkg',
@@ -199,27 +236,41 @@ class jobsManager:
                 f_to_upload.append(sigpath)
                 f_to_upload.append(fpath)
         for f in f_to_upload:
-            size = f.stat().st_size / 1000 / 1000
+            max_tries = 5
-            if f.name.endswith(PKG_SUFFIX):
+            for tries in range(max_tries):
-                for _ in range(10):
+                try:
-                    timeout = rrun('push_start', args=(f.name, size))
+                    size = f.stat().st_size / 1000 / 1000
-                    if timeout > 0:
+                    if f.name.endswith(PKG_SUFFIX):
-                        break
+                        for _ in range(10):
+                            timeout = rrun('push_start', args=(f.name, size))
+                            if timeout > 0:
+                                break
+                            else:
+                                logger.warning('Remote is busy (-1), wait 1 min x10')
+                                sleep(60)
                     else:
-                        logger.warning('Remote is busy (-1), wait 1 min x10')
+                        timeout = 60
-                        sleep(60)
+                    logger.info(f'Uploading {f}, timeout in {timeout}s')
-            else:
+                    mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
-                timeout = 60
+                    if f.name.endswith(PKG_SUFFIX):
-            logger.info(f'Uploading {f}, timeout in {timeout}s')
+                        logger.info(f'Requesting repo update for {f.name}')
-            mon_bash(UPLOAD_CMD.format(src=f), seconds=int(timeout))
+                        res = rrun('push_done', args=(f.name,), kwargs={'overwrite': False,})
-            if f.name.endswith(PKG_SUFFIX):
+                        if res is None:
-                logger.info(f'Requesting repo update for {f.name}')
+                            logger.info(f'Update success for {f.name}')
-                res = rrun('push_done', args=(f.name,), kwargs={'overwrite': False,})
+                        else:
-                if res is None:
+                            logger.error(f'Update failed for {f.name}, reason: {res}')
-                    logger.info(f'Update success for {f.name}')
+                            suc = False
+                except Exception:
+                    time_to_sleep = (tries + 1) * 60
+                    logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
+                    print_exc_plus()
+                    if tries + 1 < max_tries:
+                        sleep(time_to_sleep)
                 else:
-                    logger.error(f'Update failed for {f.name}, reason: {res}')
+                    break
-                    suc = False
+            else:
+                logger.error(f'Upload {f} failed, abort.')
+                raise RuntimeError('Unable to upload some files')
         return suc
     def tick(self):
         '''
@@ -232,24 +283,25 @@ class jobsManager:
             if time() - self.last_updatecheck <= UPDATE_INTERVAL * 60:
                 if not self.idle:
                     logger.info('Buildbot is idling for package updates.')
-                self.idle = True
+                    self.idle = True
-                sleep(60)
+                return 60
-                return
+            else:
-            self.last_updatecheck = time()
+                self.last_updatecheck = time()
-            self.idle = False
+                self.idle = False
-            # git pull repo
+                # git pull repo
-            try:
+                try:
-                bash(GIT_PULL, cwd=REPO_ROOT)
+                    bash(GIT_PULL, cwd=REPO_ROOT)
-            except Exception:
+                except Exception:
-                print_exc_plus()
+                    print_exc_plus()
-            self.pkgconfigs = load_all_yaml()
+                self.pkgconfigs = load_all_yaml()
-            updates = updmgr.check_update()
+                updates = updmgr.check_update()
-            for update in updates:
+                for update in updates:
-                (pkgconfig, ver, buildarchs) = update
+                    (pkgconfig, ver, buildarchs) = update
-                march = True if len(buildarchs) >= 2 else False
+                    march = True if len(buildarchs) >= 2 else False
-                for arch in buildarchs:
+                    for arch in buildarchs:
-                    newjob = Job(arch, pkgconfig, ver, multiarch=march)
+                        newjob = Job(arch, pkgconfig, ver, multiarch=march)
-                    self._new_buildjob(newjob)
+                        self._new_buildjob(newjob)
+                return 0
         else:
             # This part does the job
             self.idle = False
@@ -272,35 +324,41 @@ class jobsManager:
                     else:
                         self.__clean(job, rm_src=False, remove_pkg=True)
             self.__finish_job(job.pkgconfig.dirname)
+            return 0
+
 jobsmgr = jobsManager()
 
 class updateManager:
     def __init__(self, filename='pkgver.json'):
         self.__filename = filename
+        self.__pkgerrs = dict()
         self.__pkgvers = dict()
         self.__load()
     def __load(self):
         if Path(self.__filename).exists():
             with open(self.__filename,"r") as f:
                 try:
-                    pkgvers = json.loads(f.read())
+                    pkgdata = json.loads(f.read())
                 except json.JSONDecodeError:
                     logger.error('pkgver.json - Bad json')
                     print_exc_plus
                     exit(1)
         else:
             logger.warning(f'No {self.__filename} found')
-            pkgvers = dict()
+            pkgdata = dict()
-        assert type(pkgvers) is dict
+        assert type(pkgdata) is dict
-        for pkgname in pkgvers:
+        for pkgname in pkgdata:
             assert type(pkgname) is str
-        self.__pkgvers = pkgvers
+            assert len(pkgdata[pkgname]) == 2
+        self.__pkgvers = {pkgname:pkgdata[pkgname][0] for pkgname in pkgdata}
+        self.__pkgerrs = {pkgname:pkgdata[pkgname][1] for pkgname in pkgdata}
     def _save(self):
-        pkgvers = json.dumps(self.__pkgvers, indent=4)
+        pkgdata = {pkgname:[self.__pkgvers[pkgname], self.__pkgerrs[pkgname]] for pkgname in self.__pkgvers}
-        pkgvers += '\n'
+        pkgdatastr = json.dumps(pkgdata, indent=4)
+        pkgdatastr += '\n'
         with open(self.__filename,"w") as f:
             if f.writable:
-                f.write(pkgvers)
+                f.write(pkgdatastr)
             else:
                 logger.error('pkgver.json - Not writable')
     def __get_package_list(self, dirname, arch):
@@ -332,12 +390,21 @@ class updateManager:
                     continue
                 # hopefully we only need to check one arch for update
                 arch = 'x86_64' if 'x86_64' in buildarchs else buildarchs[0] # prefer x86
+                # run pre_update_scripts
+                logger.info('running pre-update scripts')
+                for scr in getattr(pkg, 'update', list()):
+                    if type(scr) is str:
+                        mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60)
                 mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
                                 logfile = pkgdir / 'buildbot.log.update',
                                 short_return = True)
                 if pkg.type in ('git', 'manual'):
                     ver = self.__get_new_ver(pkg.dirname, arch)
                     oldver = self.__pkgvers.get(pkg.dirname, None)
+                    errs = self.__pkgerrs.get(pkg.dirname, 0)
+                    if errs >= 2:
+                        logger.warning(f'package: {pkg.dirname} too many failures checking update')
+                        continue
                     has_update = False
                     if rebuild_package:
                         has_update = True
@@ -353,10 +420,12 @@ class updateManager:
                         has_update = True
                     if has_update:
                         self.__pkgvers[pkg.dirname] = ver
+                        self.__pkgerrs[pkg.dirname] = 0
                         updates.append((pkg, ver, buildarchs))
                 else:
                     logger.warning(f'unknown package type: {pkg.type}')
             except Exception:
+                self.__pkgerrs[pkg.dirname] = self.__pkgerrs.get(pkg.dirname, 0) + 1
                 print_exc_plus()
         self._save()
         return updates
@@ -414,11 +483,23 @@ if __name__ == '__main__':
     logger.info('Listener started.')
     while True:
         try:
-            jobsmgr.tick()
+            ret = 1
+            ret = jobsmgr.tick()
         except Exception:
+            try:
+                jobsmgr.clean_failed_job()
+            except Exception:
+                print_exc_plus()
             print_exc_plus()
         except KeyboardInterrupt:
             logger.info('KeyboardInterrupt')
             print_exc_plus()
             break
-        sleep(1)
+        if ret is None:
+            sleep(1)
+        elif ret == 0:
+            pass
+        elif type(ret) in (int, float):
+            sleep(ret)
+        else:
+            sleep(1)

extra.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import os
+import logging
+from pathlib import Path
+
+from utils import print_exc_plus
+
+logger = logging.getLogger(f'buildbot.{__name__}')
+
+abspath=os.path.abspath(__file__)
+abspath=os.path.dirname(abspath)
+os.chdir(abspath)

notify.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# notify.py: Automatic management tool for an arch repo.
+# This file is part of Buildbot by JerryXiao
+
+
+import logging
+from utils import background, print_exc_plus, configure_logger
+
+logger = logging.getLogger(f'buildbot.{__name__}')
+
+# wip
+# does nothing
+@background
+def send(content):
+    pass

utils.py

@@ -261,11 +261,19 @@ def format_exc_plus():
 
 def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
                      level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
+    try:
+        from notify import send
+    except ModuleNotFoundError:
+        def send(*args):
+            pass
+
     class ExceptionFormatter(logging.Formatter):
         def format(self, record):
             if record.levelno == 49:
                 record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
-            return super().format(record)
+            fmtr = super().format(record)
+            send(fmtr)
+            return fmtr
 
     logger.setLevel(logging.DEBUG)
     formatter = ExceptionFormatter(fmt=format)