buildbot: add extra

buildbot.py

@@ -19,7 +19,8 @@ from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
                    MAKEPKG_MAKE_CMD, MAKEPKG_MAKE_CMD_CLEAN, \
                    GPG_SIGN_CMD, GPG_VERIFY_CMD, UPDATE_INTERVAL, \
                    MAKEPKG_MAKE_CMD_MARCH, UPLOAD_CMD, \
-                   GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE
+                   GIT_PULL, GIT_RESET_SUBDIR, CONSOLE_LOGFILE, \
+                   MAIN_LOGFILE, PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE
 
 from utils import print_exc_plus, background, \
                   bash, get_pkg_details_from_name, vercmp, \
@@ -32,12 +33,16 @@ import json
 
 from yamlparse import load_all as load_all_yaml
 
+from extra import gen_pkglist as extra_gen_pkglist, \
+                  readpkglog as extra_readpkglog, \
+                  readmainlog as extra_readmainlog
+
 abspath=os.path.abspath(__file__)
 abspath=os.path.dirname(abspath)
 os.chdir(abspath)
 
 logger = logging.getLogger('buildbot')
-configure_logger(logger, logfile='buildbot.log', rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE)
+configure_logger(logger, logfile=MAIN_LOGFILE, rotate_size=1024*1024*10, enable_notify=True, consolelog=CONSOLE_LOGFILE)
 
 # refuse to run in systemd-nspawn
 if 'systemd-nspawn' in bash('systemd-detect-virt || true'):
@@ -214,7 +219,7 @@ class jobsManager:
         # actually makepkg
         try:
             ret = mon_nspawn_shell(arch=job.arch, cwd=cwd, cmdline=mkcmd,
-                                    logfile = cwd / 'buildbot.log.makepkg',
+                                    logfile = cwd / MAKEPKG_LOGFILE,
                                     short_return = True,
                                     seconds=job.pkgconfig.timeout*60)
         except Exception:
@@ -399,6 +404,12 @@ class updateManager:
         self.__pkgerrs = dict()
         self.__pkgvers = dict()
         self.__load()
+    @property
+    def pkgvers(self):
+        return self.__pkgvers
+    @property
+    def pkgerrs(self):
+        return self.__pkgerrs
     def __load(self):
         if Path(self.__filename).exists():
             with open(self.__filename,"r") as f:
@@ -465,7 +476,7 @@ class updateManager:
                     if type(scr) is str:
                         mon_nspawn_shell(arch, scr, cwd=pkgdir, seconds=60*60)
                 mon_nspawn_shell(arch, MAKEPKG_UPD_CMD, cwd=pkgdir, seconds=60*60,
-                                logfile = pkgdir / 'buildbot.log.update',
+                                logfile = pkgdir / PKG_UPDATE_LOGFILE,
                                 short_return = True)
                 if pkg.type in ('git', 'manual'):
                     ver = self.__get_new_ver(pkg.dirname, arch)
@@ -532,6 +543,24 @@ def force_upload(pkgdirname, overwrite=False):
 def getup():
     return jobsmgr.getup()
 
+def extras(action, pkgname=None):
+    if action.startswith("pkg"):
+        p = extra_gen_pkglist(jobsmgr.pkgconfigs, updmgr.pkgvers, updmgr.pkgerrs)
+        if action == "pkgdetail":
+            return p[1].get(pkgname, None)
+        elif action == "pkgdetails":
+            return p[1]
+        elif action == "pkglist":
+            return p[0]
+    elif action == "mainlog":
+        return extra_readmainlog(debug=False)
+    elif action == "debuglog":
+        return extra_readmainlog(debug=True)
+    elif action == "readpkglog":
+        pkgname = str(pkgname)
+        return extra_readpkglog(pkgname)
+    return False
+
 def run(funcname, args=list(), kwargs=dict()):
     if funcname in ('info', 'rebuild_package', 'clean', 'clean_all',
                     'force_upload', 'getup'):

client.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# repod.py: Automatic management tool for an arch repo.
+# client.py: Automatic management tool for an arch repo.
 # This file is part of Buildbot by JerryXiao
 
 import logging
@@ -10,7 +10,7 @@ from time import sleep
 
 from config import REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD, \
                    MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
-                   CONSOLE_LOGFILE
+                   CONSOLE_LOGFILE, MAIN_LOGFILE
 
 from utils import print_exc_plus
 
@@ -47,7 +47,7 @@ if __name__ == '__main__':
     configure_logger(logger)
     def print_log(debug=False):
         if debug:
-            os.system(f'tail -n 43 -f \"buildbot.log\"')
+            os.system(f'tail -n 43 -f \"{MAIN_LOGFILE}\"')
         else:
             os.system(f'tail -n 43 -f \"{CONSOLE_LOGFILE}\"')
     try:

config.py

@@ -63,4 +63,9 @@ UPLOAD_CMD = 'rsync -avPh \"{src}\" repoupload:/srv/repo/buildbot/repo/updates/'
 GIT_PULL = 'git pull'
 GIT_RESET_SUBDIR = 'git checkout HEAD -- .'
 
+
+# logfiles
+MAIN_LOGFILE = 'buildbot.log'
 CONSOLE_LOGFILE = 'buildbot.log.console'
+PKG_UPDATE_LOGFILE = 'buildbot.log.update'
+MAKEPKG_LOGFILE = 'buildbot.log.makepkg'

extra.py

@@ -1,13 +1,62 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+# extra.py: Automatic management tool for an arch repo.
+# This file is part of Buildbot by JerryXiao
+
 import os
 import logging
-from pathlib import Path
 
+from pathlib import Path
 from utils import print_exc_plus
 
+from config import PKGBUILD_DIR, MAIN_LOGFILE, CONSOLE_LOGFILE, \
+                   PKG_UPDATE_LOGFILE, MAKEPKG_LOGFILE
+
 logger = logging.getLogger(f'buildbot.{__name__}')
 
 abspath=os.path.abspath(__file__)
 abspath=os.path.dirname(abspath)
 os.chdir(abspath)
+
+REPO_ROOT = Path(PKGBUILD_DIR)
+
+# generate package list
+def gen_pkglist(pkgconfigs, pkgvers, pkgerrs):
+    # pkgall contains details
+    # namelist is a list of pkgnames
+    pkgall = dict()
+    for pc in pkgconfigs:
+        ps = ('type', 'cleanbuild', 'timeout')
+        hps = ('prebuild', 'postbuild', 'update', 'failure')
+        dps = {p:getattr(pc, p, None) for p in ps}
+        dhps = {p:'\n'.join(str(getattr(pc, p, None))) for p in hps}
+        # additional package details
+        ves = {'version': pkgvers.get(pc.dirname, None), 'errors': pkgerrs.get(pc.dirname, None)}
+        pkgall[pc.dirname] = {**dps, **dhps, **ves}
+    namelist = [k for k in pkgall]
+    return (namelist, pkgall)
+
+def __simpleread(fpath, limit=4096-100):
+    with open(fpath, 'r') as f:
+        c = f.read()
+    if len(c) > limit:
+        c = c[-limit:]
+    return c
+# read logs
+def readpkglog(pkgdirname, update=False):
+    cwd = REPO_ROOT / pkgdirname
+    logfile = PKG_UPDATE_LOGFILE if update else MAKEPKG_LOGFILE
+    if cwd.exists() and (cwd / logfile).exists():
+        logger.debug(f'formatting {"update" if update else "build"} logs in {pkgdirname}')
+        return __simpleread(cwd / logfile)
+    else:
+        logger.debug(f'not found: {"update" if update else "build"} log in dir {pkgdirname}')
+        return f"{cwd / logfile} cannot be found"
+def readmainlog(debug=False):
+    logfile = MAIN_LOGFILE if debug else CONSOLE_LOGFILE
+    if (PKGBUILD_DIR / logfile).exists():
+        logger.debug(f'formatting buildbot{" debug" if debug else ""} logs')
+        return __simpleread(PKGBUILD_DIR / logfile)
+    else:
+        logger.debug(f'not found: buildbot{" debug" if debug else ""} log')
+        return f"{PKGBUILD_DIR / logfile} cannot be found"