diff --git a/buildbot.py b/buildbot.py index dbb7139..7d7a98b 100755 --- a/buildbot.py +++ b/buildbot.py @@ -109,7 +109,11 @@ class jobsManager: else: return False return True + def force_upload_package(self, pkgdirname, overwrite=True): + pass def rebuild_package(self, pkgdirname, clean=False): + if not self.idle: + logger.debug('rebuild requested and not idle.') self.pkgconfigs = load_all_yaml() if (REPO_ROOT / pkgdirname).exists() and clean: self.reset_dir(pkgdirname) @@ -263,6 +267,8 @@ class jobsManager: except Exception: time_to_sleep = (tries + 1) * 60 logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs') + if not rrun('push_fail', args=(f.name,)): + logger.error('unable to run push_fail') print_exc_plus() if tries + 1 < max_tries: sleep(time_to_sleep) @@ -272,6 +278,13 @@ class jobsManager: logger.error(f'Upload {f} failed, abort.') raise RuntimeError('Unable to upload some files') return suc + def getup(self): + ''' + check for updates now !!! + ''' + logger.info('Check for updates now.') + self.last_updatecheck = 0.0 + return "buildbot wakes up" def tick(self): ''' check for updates, @@ -450,8 +463,16 @@ def clean(pkgdirname): def clean_all(): return jobsmgr.reset_dir(all=True) +def force_upload(pkgdirname): + #jobsmgr.force_upload_package(pkgdirname) + return "not implemented" + +def getup(): + return jobsmgr.getup() + def run(funcname, args=list(), kwargs=dict()): - if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'): + if funcname in ('info', 'rebuild_package', 'clean', 'clean_all', + 'force_upload', 'getup'): logger.debug('running: %s %s %s',funcname, args, kwargs) ret = eval(funcname)(*args, **kwargs) logger.info('done: %s %s %s',funcname, args, kwargs) @@ -466,7 +487,7 @@ def __main(): try: with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener: with listener.accept() as conn: - logger.info('connection accepted from %s', listener.last_accepted) + logger.debug('connection accepted from %s', listener.last_accepted) myrecv = conn.recv() if type(myrecv) is list and len(myrecv) == 3: (funcname, args, kwargs) = myrecv diff --git a/client.py b/client.py index b1fa64e..80b10e1 100755 --- a/client.py +++ b/client.py @@ -71,30 +71,70 @@ if __name__ == '__main__': lines = list() sleep(1) try: - parser = argparse.ArgumentParser(description='Client for buildbot') - parser.add_argument('--info', action='store_true', help='show buildbot info') - parser.add_argument('--update', action='store_true', help='update pushed files to the repo') - parser.add_argument('--cleanall', action='store_true', help='checkout pkgbuilds') - parser.add_argument('--clean', nargs='?', default=None, help='checkout pkgbuilds in one package') - parser.add_argument('--rebuild', nargs='?', default=None, help='rebuild a package with its dirname') - parser.add_argument('--log', action='store_true' , help='print log') + actions = { + 'info': 'show buildbot info', + 'update': '[--overwrite] update pushed files to the repo', + 'clean': '[dir / all] checkout pkgbuilds in packages', + 'rebuild': '[dir1 dir2 --clean] rebuild packages', + 'log': '[--debug] print log', + 'upload': '[dir1 dir2] force upload packages', + 'getup': 'check for updates now' + } + parser = argparse.ArgumentParser(description='Client for buildbot', + formatter_class=argparse.RawTextHelpFormatter) + __action_help = "\n".join([f"{a}:\t{actions[a]}" for a in actions]) + parser.add_argument('action', nargs='*', help=f'Choose which action to invoke:\n\n{__action_help}') + parser.add_argument('--overwrite', nargs='?', default='False', help='overwrite existing files') + parser.add_argument('--debug', nargs='?', default='False', help='print debug logs') + parser.add_argument('--clean', nargs='?', default='True', help='clean build packages') args = parser.parse_args() - if args.info: + action = args.action + for switch in ('overwrite', 'debug', 'clean'): + s = getattr(args, switch) + if str(s).lower() in ('false', 'no', 'n', '0'): + setattr(args, switch, False) + else: + setattr(args, switch, True) + if not (action and len(action) >= 1 and action[0] in actions): + parser.print_help() + parser.exit(status=1) + if action[0] == 'info': server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD) logger.info(run('info', server=server)) - elif args.update: + elif action[0] == 'getup': + server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD) + logger.info(run('getup', server=server)) + elif action[0] == 'update': server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD) logger.info(run('update', kwargs={'overwrite': False}, server=server)) - elif args.cleanall: + elif action[0] == 'clean': + if len(action) <= 1: + print('Error: Need package name') + parser.print_help() + parser.exit(status=1) server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD) - logger.info(run('clean_all', server=server)) - elif args.clean: + if 'all' in action[1:]: + logger.info(run('clean_all', server=server)) + else: + for p in action[1:]: + logger.info(run('clean', args=(p,), server=server)) + elif action[0] == 'rebuild': + if len(action) <= 1: + print('Error: Need package name') + parser.print_help() + parser.exit(status=1) server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD) - logger.info(run('clean', args=(args.clean,), server=server)) - elif args.rebuild: + for p in action[1:]: + logger.info(run('rebuild_package', args=(p,), kwargs={'clean': args.clean}, server=server)) + elif action[0] == 'upload': + if len(action) <= 1: + print('Error: Need package name') + parser.print_help() + parser.exit(status=1) server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD) - logger.info(run('rebuild_package', args=(args.rebuild,), kwargs={'clean': True}, server=server)) - elif args.log: + for p in action[1:]: + logger.info(run('force_upload', args=(p,), server=server)) + elif action[0] == 'log': logger.info('printing logs') print_log() else: diff --git a/repod.py b/repod.py index 1c7e66a..31eabb6 100755 --- a/repod.py +++ b/repod.py @@ -68,6 +68,21 @@ class pushFm: return None else: return None + def fail(self, fname): + update_path = Path('updates') + if fname == self.fname: + pkg = update_path / self.fname + sig = update_path / f'{self.fname}.sig' + for f in (pkg, sig): + if f.exists(): + try: + f.unlink() + except Exception: + logger.warning(f'unable to remove {f.name}') + self.__init__() + return None + else: + return "Wrong file" def done(self, fname, overwrite=False): ''' return None means success @@ -117,13 +132,15 @@ def push_start(filename, size): def push_done(filename, overwrite=False): return pfm.done(filename, overwrite=overwrite) - +def push_fail(filename): + return pfm.fail(filename) # server part def run(funcname, args=list(), kwargs=dict()): if funcname in ('clean', 'regenerate', 'remove', - 'update', 'push_start', 'push_done'): + 'update', 'push_start', 'push_done', + 'push_fail'): logger.info('running: %s %s %s', funcname, args, kwargs) ret = eval(funcname)(*args, **kwargs) logger.info('done: %s %s',funcname, ret) diff --git a/utils.py b/utils.py index e2fa6d4..d153328 100644 --- a/utils.py +++ b/utils.py @@ -260,23 +260,36 @@ def format_exc_plus(): return ret def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s', - level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None): - try: - from notify import send - except ModuleNotFoundError: - def send(*args): - pass + level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None, + enable_notify=False): + def __send(*args): + pass + if enable_notify: + try: + from notify import send + except ModuleNotFoundError: + send = __send + else: + send = __send class ExceptionFormatter(logging.Formatter): + def __init__(self, *args, notify=False, **kwargs): + self.notify = notify + super().__init__(*args, **kwargs) def format(self, record): if record.levelno == 49: record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg fmtr = super().format(record) - send(fmtr) + if self.notify: + send(fmtr) return fmtr logger.setLevel(logging.DEBUG) - formatter = ExceptionFormatter(fmt=format) + fnotify = cnotify = False + fnotify = True if enable_notify and logfile else False + cnotify = True if enable_notify and (not logfile) else False + fformatter = ExceptionFormatter(fmt=format, notify=fnotify) + cformatter = ExceptionFormatter(fmt=format, notify=cnotify) logging.addLevelName(49, 'Exception') # create file handler if logfile: @@ -287,10 +300,10 @@ def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - else: fh = logging.FileHandler(logfile) fh.setLevel(flevel) - fh.setFormatter(formatter) + fh.setFormatter(fformatter) logger.addHandler(fh) # create console handler ch = logging.StreamHandler() ch.setLevel(level) - ch.setFormatter(formatter) + ch.setFormatter(cformatter) logger.addHandler(ch) diff --git a/yamlparse.py b/yamlparse.py index d790621..584029f 100644 --- a/yamlparse.py +++ b/yamlparse.py @@ -73,7 +73,7 @@ def load_all(): pkgconfigs = list() for mydir in REPO_ROOT.iterdir(): try: - if mydir.is_dir(): + if mydir.is_dir() and (not mydir.name.startswith('.')): if (mydir / AUTOBUILD_FNAME).exists(): # parsing yaml logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME, mydir)