mirror of
https://github.com/archlinux-jerry/buildbot
synced 2024-12-05 03:10:41 +08:00
buildbot.py: fix bugs and rewrite client
This commit is contained in:
parent
61bbd523f4
commit
e68c227ec5
5 changed files with 122 additions and 31 deletions
25
buildbot.py
25
buildbot.py
|
@ -109,7 +109,11 @@ class jobsManager:
|
|||
else:
|
||||
return False
|
||||
return True
|
||||
def force_upload_package(self, pkgdirname, overwrite=True):
|
||||
pass
|
||||
def rebuild_package(self, pkgdirname, clean=False):
|
||||
if not self.idle:
|
||||
logger.debug('rebuild requested and not idle.')
|
||||
self.pkgconfigs = load_all_yaml()
|
||||
if (REPO_ROOT / pkgdirname).exists() and clean:
|
||||
self.reset_dir(pkgdirname)
|
||||
|
@ -263,6 +267,8 @@ class jobsManager:
|
|||
except Exception:
|
||||
time_to_sleep = (tries + 1) * 60
|
||||
logger.error(f'We are getting problem uploading {f}, wait {time_to_sleep} secs')
|
||||
if not rrun('push_fail', args=(f.name,)):
|
||||
logger.error('unable to run push_fail')
|
||||
print_exc_plus()
|
||||
if tries + 1 < max_tries:
|
||||
sleep(time_to_sleep)
|
||||
|
@ -272,6 +278,13 @@ class jobsManager:
|
|||
logger.error(f'Upload {f} failed, abort.')
|
||||
raise RuntimeError('Unable to upload some files')
|
||||
return suc
|
||||
def getup(self):
|
||||
'''
|
||||
check for updates now !!!
|
||||
'''
|
||||
logger.info('Check for updates now.')
|
||||
self.last_updatecheck = 0.0
|
||||
return "buildbot wakes up"
|
||||
def tick(self):
|
||||
'''
|
||||
check for updates,
|
||||
|
@ -450,8 +463,16 @@ def clean(pkgdirname):
|
|||
def clean_all():
|
||||
return jobsmgr.reset_dir(all=True)
|
||||
|
||||
def force_upload(pkgdirname):
|
||||
#jobsmgr.force_upload_package(pkgdirname)
|
||||
return "not implemented"
|
||||
|
||||
def getup():
|
||||
return jobsmgr.getup()
|
||||
|
||||
def run(funcname, args=list(), kwargs=dict()):
|
||||
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all'):
|
||||
if funcname in ('info', 'rebuild_package', 'clean', 'clean_all',
|
||||
'force_upload', 'getup'):
|
||||
logger.debug('running: %s %s %s',funcname, args, kwargs)
|
||||
ret = eval(funcname)(*args, **kwargs)
|
||||
logger.info('done: %s %s %s',funcname, args, kwargs)
|
||||
|
@ -466,7 +487,7 @@ def __main():
|
|||
try:
|
||||
with Listener(MASTER_BIND_ADDRESS, authkey=MASTER_BIND_PASSWD) as listener:
|
||||
with listener.accept() as conn:
|
||||
logger.info('connection accepted from %s', listener.last_accepted)
|
||||
logger.debug('connection accepted from %s', listener.last_accepted)
|
||||
myrecv = conn.recv()
|
||||
if type(myrecv) is list and len(myrecv) == 3:
|
||||
(funcname, args, kwargs) = myrecv
|
||||
|
|
70
client.py
70
client.py
|
@ -71,30 +71,70 @@ if __name__ == '__main__':
|
|||
lines = list()
|
||||
sleep(1)
|
||||
try:
|
||||
parser = argparse.ArgumentParser(description='Client for buildbot')
|
||||
parser.add_argument('--info', action='store_true', help='show buildbot info')
|
||||
parser.add_argument('--update', action='store_true', help='update pushed files to the repo')
|
||||
parser.add_argument('--cleanall', action='store_true', help='checkout pkgbuilds')
|
||||
parser.add_argument('--clean', nargs='?', default=None, help='checkout pkgbuilds in one package')
|
||||
parser.add_argument('--rebuild', nargs='?', default=None, help='rebuild a package with its dirname')
|
||||
parser.add_argument('--log', action='store_true' , help='print log')
|
||||
actions = {
|
||||
'info': 'show buildbot info',
|
||||
'update': '[--overwrite] update pushed files to the repo',
|
||||
'clean': '[dir / all] checkout pkgbuilds in packages',
|
||||
'rebuild': '[dir1 dir2 --clean] rebuild packages',
|
||||
'log': '[--debug] print log',
|
||||
'upload': '[dir1 dir2] force upload packages',
|
||||
'getup': 'check for updates now'
|
||||
}
|
||||
parser = argparse.ArgumentParser(description='Client for buildbot',
|
||||
formatter_class=argparse.RawTextHelpFormatter)
|
||||
__action_help = "\n".join([f"{a}:\t{actions[a]}" for a in actions])
|
||||
parser.add_argument('action', nargs='*', help=f'Choose which action to invoke:\n\n{__action_help}')
|
||||
parser.add_argument('--overwrite', nargs='?', default='False', help='overwrite existing files')
|
||||
parser.add_argument('--debug', nargs='?', default='False', help='print debug logs')
|
||||
parser.add_argument('--clean', nargs='?', default='True', help='clean build packages')
|
||||
args = parser.parse_args()
|
||||
if args.info:
|
||||
action = args.action
|
||||
for switch in ('overwrite', 'debug', 'clean'):
|
||||
s = getattr(args, switch)
|
||||
if str(s).lower() in ('false', 'no', 'n', '0'):
|
||||
setattr(args, switch, False)
|
||||
else:
|
||||
setattr(args, switch, True)
|
||||
if not (action and len(action) >= 1 and action[0] in actions):
|
||||
parser.print_help()
|
||||
parser.exit(status=1)
|
||||
if action[0] == 'info':
|
||||
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
|
||||
logger.info(run('info', server=server))
|
||||
elif args.update:
|
||||
elif action[0] == 'getup':
|
||||
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
|
||||
logger.info(run('getup', server=server))
|
||||
elif action[0] == 'update':
|
||||
server=(REPOD_BIND_ADDRESS, REPOD_BIND_PASSWD)
|
||||
logger.info(run('update', kwargs={'overwrite': False}, server=server))
|
||||
elif args.cleanall:
|
||||
elif action[0] == 'clean':
|
||||
if len(action) <= 1:
|
||||
print('Error: Need package name')
|
||||
parser.print_help()
|
||||
parser.exit(status=1)
|
||||
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
|
||||
if 'all' in action[1:]:
|
||||
logger.info(run('clean_all', server=server))
|
||||
elif args.clean:
|
||||
else:
|
||||
for p in action[1:]:
|
||||
logger.info(run('clean', args=(p,), server=server))
|
||||
elif action[0] == 'rebuild':
|
||||
if len(action) <= 1:
|
||||
print('Error: Need package name')
|
||||
parser.print_help()
|
||||
parser.exit(status=1)
|
||||
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
|
||||
logger.info(run('clean', args=(args.clean,), server=server))
|
||||
elif args.rebuild:
|
||||
for p in action[1:]:
|
||||
logger.info(run('rebuild_package', args=(p,), kwargs={'clean': args.clean}, server=server))
|
||||
elif action[0] == 'upload':
|
||||
if len(action) <= 1:
|
||||
print('Error: Need package name')
|
||||
parser.print_help()
|
||||
parser.exit(status=1)
|
||||
server=(MASTER_BIND_ADDRESS, MASTER_BIND_PASSWD)
|
||||
logger.info(run('rebuild_package', args=(args.rebuild,), kwargs={'clean': True}, server=server))
|
||||
elif args.log:
|
||||
for p in action[1:]:
|
||||
logger.info(run('force_upload', args=(p,), server=server))
|
||||
elif action[0] == 'log':
|
||||
logger.info('printing logs')
|
||||
print_log()
|
||||
else:
|
||||
|
|
21
repod.py
21
repod.py
|
@ -68,6 +68,21 @@ class pushFm:
|
|||
return None
|
||||
else:
|
||||
return None
|
||||
def fail(self, fname):
|
||||
update_path = Path('updates')
|
||||
if fname == self.fname:
|
||||
pkg = update_path / self.fname
|
||||
sig = update_path / f'{self.fname}.sig'
|
||||
for f in (pkg, sig):
|
||||
if f.exists():
|
||||
try:
|
||||
f.unlink()
|
||||
except Exception:
|
||||
logger.warning(f'unable to remove {f.name}')
|
||||
self.__init__()
|
||||
return None
|
||||
else:
|
||||
return "Wrong file"
|
||||
def done(self, fname, overwrite=False):
|
||||
'''
|
||||
return None means success
|
||||
|
@ -117,13 +132,15 @@ def push_start(filename, size):
|
|||
def push_done(filename, overwrite=False):
|
||||
return pfm.done(filename, overwrite=overwrite)
|
||||
|
||||
|
||||
def push_fail(filename):
|
||||
return pfm.fail(filename)
|
||||
|
||||
# server part
|
||||
|
||||
def run(funcname, args=list(), kwargs=dict()):
|
||||
if funcname in ('clean', 'regenerate', 'remove',
|
||||
'update', 'push_start', 'push_done'):
|
||||
'update', 'push_start', 'push_done',
|
||||
'push_fail'):
|
||||
logger.info('running: %s %s %s', funcname, args, kwargs)
|
||||
ret = eval(funcname)(*args, **kwargs)
|
||||
logger.info('done: %s %s',funcname, ret)
|
||||
|
|
25
utils.py
25
utils.py
|
@ -260,23 +260,36 @@ def format_exc_plus():
|
|||
return ret
|
||||
|
||||
def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s - %(message)s',
|
||||
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None):
|
||||
level=logging.INFO, logfile=None, flevel=logging.DEBUG, rotate_size=None,
|
||||
enable_notify=False):
|
||||
def __send(*args):
|
||||
pass
|
||||
if enable_notify:
|
||||
try:
|
||||
from notify import send
|
||||
except ModuleNotFoundError:
|
||||
def send(*args):
|
||||
pass
|
||||
send = __send
|
||||
else:
|
||||
send = __send
|
||||
|
||||
class ExceptionFormatter(logging.Formatter):
|
||||
def __init__(self, *args, notify=False, **kwargs):
|
||||
self.notify = notify
|
||||
super().__init__(*args, **kwargs)
|
||||
def format(self, record):
|
||||
if record.levelno == 49:
|
||||
record.msg = 'Exception caught.\nPrinting stack traceback\n' + record.msg
|
||||
fmtr = super().format(record)
|
||||
if self.notify:
|
||||
send(fmtr)
|
||||
return fmtr
|
||||
|
||||
logger.setLevel(logging.DEBUG)
|
||||
formatter = ExceptionFormatter(fmt=format)
|
||||
fnotify = cnotify = False
|
||||
fnotify = True if enable_notify and logfile else False
|
||||
cnotify = True if enable_notify and (not logfile) else False
|
||||
fformatter = ExceptionFormatter(fmt=format, notify=fnotify)
|
||||
cformatter = ExceptionFormatter(fmt=format, notify=cnotify)
|
||||
logging.addLevelName(49, 'Exception')
|
||||
# create file handler
|
||||
if logfile:
|
||||
|
@ -287,10 +300,10 @@ def configure_logger(logger, format='%(asctime)s - %(name)-18s - %(levelname)s -
|
|||
else:
|
||||
fh = logging.FileHandler(logfile)
|
||||
fh.setLevel(flevel)
|
||||
fh.setFormatter(formatter)
|
||||
fh.setFormatter(fformatter)
|
||||
logger.addHandler(fh)
|
||||
# create console handler
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(level)
|
||||
ch.setFormatter(formatter)
|
||||
ch.setFormatter(cformatter)
|
||||
logger.addHandler(ch)
|
||||
|
|
|
@ -73,7 +73,7 @@ def load_all():
|
|||
pkgconfigs = list()
|
||||
for mydir in REPO_ROOT.iterdir():
|
||||
try:
|
||||
if mydir.is_dir():
|
||||
if mydir.is_dir() and (not mydir.name.startswith('.')):
|
||||
if (mydir / AUTOBUILD_FNAME).exists():
|
||||
# parsing yaml
|
||||
logger.info('Bulidbot: found %s in %s', AUTOBUILD_FNAME, mydir)
|
||||
|
|
Loading…
Reference in a new issue