buildbot/buildbot.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# buildbot.py: Automatic management tool for an arch repo.
# This file is part of Buildbot by JerryXiao

import logging
from multiprocessing.connection import Listener
from time import time, sleep
import os
from pathlib import Path
from subprocess import CalledProcessError

from utils import print_exc_plus, background

from config import ARCHS, BUILD_ARCHS, BUILD_ARCH_MAPPING, \
                   MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD, \
                   PKGBUILD_DIR, MAKEPKG_PKGLIST_CMD, MAKEPKG_UPD_CMD
from utils import bash, get_pkg_details_from_name, vercmp

import json

from yamlparse import load_all as load_all_yaml

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

abspath=os.path.abspath(__file__)
abspath=os.path.dirname(abspath)
os.chdir(abspath)

REPO_ROOT = Path(PKGBUILD_DIR)

class Job:
    def __init__(self, arch, pkgdir, packagelist, version):
        buildarch = BUILD_ARCH_MAPPING.get(arch, None)
        assert buildarch in BUILD_ARCHS
        self.arch = arch
        self.buildarch = buildarch
        self.pkgdir = pkgdir
        self.packagelist = packagelist
        self.version = version
        self.added = time()
        self.claimed = 0

class jobsManager:
    def __init__(self):
        self.__buildjobs = dict()
        for arch in BUILD_ARCHS:
            self.__buildjobs.setdefault(arch, list())
        self.__uploadjobs = list()
        self.__curr_job = None
        self.pkgconfigs = load_all_yaml()
    def _new_buildjob(self, job, buildarch):
        assert type(job) is Job
        self.__buildjobs.get(buildarch).append(job)
    def claim_job(self, buildarch):
        assert buildarch in BUILD_ARCHS
        if self.__curr_job:
            return None
        jobs = self.__buildjobs.get(buildarch, list())
        if jobs:
            self.__curr_job = jobs.pop(0)
            return self.__curr_job
    def __finish_job(self, pkgdir):
        assert pkgdir == self.__curr_job.pkgdir
        # do upload
        self.__curr_job = None
        return True
    def tick(self):
        '''
            check for updates,
            create new jobs
            and run them
        '''
        if self.__curr_job is None:
            updates = updmgr.check_update()
            for update in updates:
                (pkg, packagelist, ver) = update


jobsmgr = jobsManager()

class updateManager:
    def __init__(self, filename='pkgver.json'):
        self.__filename = filename
        self.__pkgvers = dict()
        self.__load()
    def __load(self):
        if Path(self.__filename).exists():
            with open(self.__filename,"r") as f:
                try:
                    pkgvers = json.loads(f.read())
                except json.JSONDecodeError:
                    logger.error('pkgver.json - Bad json')
                    print_exc_plus
                    exit(1)
        else:
            logger.warning(f'No {self.__filename} found')
            pkgvers = dict()
        assert type(pkgvers) is dict
        for pkgname in pkgvers:
            assert type(pkgname) is str
        self.__pkgvers = pkgvers
    def _save(self):
        pkgvers = json.dumps(self.__pkgvers, indent=4)
        pkgvers += '\n'
        with open(self.__filename,"w") as f:
            if f.writable:
                f.write(pkgvers)
            else:
                logger.error('pkgver.json - Not writable')
    def __get_package_list(self, dirname):
        pkgdir = REPO_ROOT / dirname
        assert pkgdir.exists()
        pkglist = bash(MAKEPKG_PKGLIST_CMD, cwd=pkgdir)
        pkglist = pkglist.split('\n')
        return pkglist
    def __get_new_ver(self, dirname):
        pkgfiles = self.__get_package_list(dirname)
        ver = get_pkg_details_from_name(pkgfiles[0])
        return (ver, pkgfiles)
    def check_update(self):
        updates = list()
        for pkg in jobsmgr.pkgconfigs:
            pkgdir = REPO_ROOT / pkg.dirname
            logger.info(f'checking update: {pkg.dirname}')
            bash(MAKEPKG_UPD_CMD, cwd=pkgdir, RUN_CMD_TIMEOUT=60*60)
            if pkg.type in ('git', 'manual'):
                (ver, pkgfiles) = self.__get_new_ver(pkg.dirname)
                oldver = self.__pkgvers.get(pkg.dirname, None)
                if oldver is None or vercmp(ver, oldver) == 1:
                    self.__pkgvers[pkg.dirname] = ver
                    updates.append((pkg, pkgfiles, ver))
                else:
                    logger.warning(f'package: {pkg.dirname} downgrade attempted')
            else:
                logger.warning(f'unknown package type: {pkg.type}')
        self._save()
        return updates

updmgr = updateManager()


@background
def __main():
    pass







def run(funcname, args=list(), kwargs=dict()):
    if funcname in ('clean', 'regenerate', 'remove',
                    'update', 'push_files', 'add_files'):
        logger.info('running: %s %s %s',funcname, args, kwargs)
        ret = eval(funcname)(*args, **kwargs)
        logger.info('done: %s %s',funcname, ret)
        return ret
    else:
        logger.error('unexpected: %s %s %s',funcname, args, kwargs)
        return False


if __name__ == '__main__':
    __main() # start the main worker thread
    while True:
        try:
            with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
                with listener.accept() as conn:
                    logger.info('connection accepted from %s', listener.last_accepted)
                    myrecv = conn.recv()
                    if type(myrecv) is list and len(myrecv) == 3:
                        (funcname, args, kwargs) = myrecv
                        funcname = str(funcname)
                        conn.send(run(funcname, args=args, kwargs=kwargs))
        except Exception:
            print_exc_plus()
        except KeyboardInterrupt:
            logger.info('KeyboardInterrupt')
            print_exc_plus()
            break