Traceback (most recent call last): File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 658, in cli_main main() File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 353, in main run_compose( File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 524, in run_compose extra_phase.stop() File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop self.pool.stop() File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process phase.stop() File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop self.pool.stop() File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process phase.stop() File "/usr/lib/python3.12/site-packages/pungi/phases/base.py", line 111, in stop self.pool.stop() File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop six.reraise(exc_info[0], exc_info[1], exc_info[2]) File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise raise value File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run self.process(item, num) File "/usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py", line 134, in process self.worker(compose, variant, config, arches, generics, repo) File "/usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py", line 178, in worker raise RuntimeError( RuntimeError: imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details Frame worker in /usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py at line 178 171 # Wait for it to finish and capture the output into log file. 172 log_dir = os.path.join(compose.paths.log.topdir(), "imageBuilderBuild") 173 util.makedirs(log_dir) 174 log_file = os.path.join( 175 log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num) 176 ) 177 if koji.watch_task(task_id, log_file) != 0: --> 178 raise RuntimeError( 179 "imageBuilderBuild task failed: %s. See %s for details" 180 % (task_id, log_file) 181 ) 182 183 # Refresh koji session which may have timed out while the task was 184 # running. Watching is done via a subprocess, so the session is arches = ['aarch64', 'ppc64le', 's390x', 'x86_64'] compose = config = {'name': 'CentOS-Stream-GenericCloud', 'version': '10', 'distro': 'centos-stream-10', 'types': ['qcow2'], 'target': 'centos-10', 'arches': ['aarch64', 'ppc64le', 's390x', 'x86_64'], 'repos': ['BaseOS' generics = {'release': '20260415.t.0', 'target': 'centos-10', 'types': ['qcow2'], 'seed': None, 'scratch': None, 'version': '10'} koji = log_dir = '/mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild' log_file = '/mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log' msg = 'imageBuilderBuild task for variant BaseOS' opts = {'repos': ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os'], ' repo = ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os'] self = self._args = self._daemonic = False self._ident = 140008839444160 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-425' self._native_id = 1989567 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failable_arches = [] self.failed = True self.get_timeout = 1 self.ident = 140008839444160 self.kill = True self.name = 'Thread-425' self.native_id = 1989567 self.num = 1 self.pool = self.running = False self.traceparent = None task_id = 6249513 variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None) Frame process in /usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py at line 134 127 compose, 128 can_task_fail, 129 variant, 130 "*", 131 "imageBuilderBuild", 132 logger=self.pool._logger, 133 ): --> 134 self.worker(compose, variant, config, arches, generics, repo) 135 136 def worker(self, compose, variant, config, arches, generics, repo): 137 msg = "imageBuilderBuild task for variant %s" % variant.uid 138 self.pool.log_info("[BEGIN] %s" % msg) 139 koji = kojiwrapper.KojiWrapper(compose) 140 koji.login() arches = ['aarch64', 'ppc64le', 's390x', 'x86_64'] can_task_fail = False compose = config = {'name': 'CentOS-Stream-GenericCloud', 'version': '10', 'distro': 'centos-stream-10', 'types': ['qcow2'], 'target': 'centos-10', 'arches': ['aarch64', 'ppc64le', 's390x', 'x86_64'], 'repos': ['BaseOS' failable_arches = [] generics = {'release': '20260415.t.0', 'target': 'centos-10', 'types': ['qcow2'], 'seed': None, 'scratch': None, 'version': '10'} item = num = 1 repo = ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os'] self = self._args = self._daemonic = False self._ident = 140008839444160 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-425' self._native_id = 1989567 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failable_arches = [] self.failed = True self.get_timeout = 1 self.ident = 140008839444160 self.kill = True self.name = 'Thread-425' self.native_id = 1989567 self.num = 1 self.pool = self.running = False self.traceparent = None variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None) Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = num = 1 self = self._args = self._daemonic = False self._ident = 140008839444160 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-425' self._native_id = 1989567 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failable_arches = [] self.failed = True self.get_timeout = 1 self.ident = 140008839444160 self.kill = True self.name = 'Thread-425' self.native_id = 1989567 self.num = 1 self.pool = self.running = False self.traceparent = None Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'), self.queue_get_lock = self.queue_processed = 2 self.queue_put_lock = self.queue_total = 2 self.threads = [, ] Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/base.py at line 111 104 "Note that variants can be excluded in configuration file" 105 ) 106 107 def stop(self): 108 if self.finished: 109 return 110 if hasattr(self, "pool"): --> 111 self.pool.stop() 112 self.finished = True 113 self.compose.log_info("[DONE ] %s" % self.msg) 114 115 if hasattr(self, "_start_time"): 116 self.compose.log_info( 117 "PHASE %s took %d seconds" self = self._skipped = False self._start_time = 1776283999.1601636 self.compose = self.finished = False self.logger = self.msg = '---------- PHASE: IMAGEBUILDER ----------' self.name = 'imagebuilder' self.pool = self.used_patterns = {'^BaseOS$'} Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81 74 self.pool.queue_total, 75 phases_names, 76 ) 77 self.pool.log_info("[BEGIN] %s" % (msg,)) 78 79 for phase in pipeline: 80 phase.start() --> 81 phase.stop() 82 83 self.pool.log_info("[DONE ] %s" % (msg,)) item = [] msg = 'Running pipeline (7/7). Phases: imagebuilder' num = 7 phase = phases_names = 'imagebuilder' pipeline = [] self = self._args = self._daemonic = False self._ident = 140008475322048 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-420' self._native_id = 1989565 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140008475322048 self.kill = True self.name = 'Thread-420' self.native_id = 1989565 self.pool = self.running = False self.traceparent = None Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = [] num = 7 self = self._args = self._daemonic = False self._ident = 140008475322048 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-420' self._native_id = 1989565 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140008475322048 self.kill = True self.name = 'Thread-420' self.native_id = 1989565 self.pool = self.running = False self.traceparent = None Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'), self.queue_get_lock = self.queue_processed = 7 self.queue_put_lock = self.queue_total = 7 self.threads = [, , , Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59 52 53 self.pool.start() 54 55 def stop(self): 56 if self.finished: 57 return 58 if hasattr(self, "pool"): --> 59 self.pool.stop() 60 self.finished = True 61 self.compose.log_info("[DONE ] %s" % self.msg) 62 63 64 class PipelineThread(WorkerThread): 65 """ self = self._phases_schema = self.compose = self.finished = False self.msg = '---------- PHASE: WEAVER ----------' self.name = 'weaver' self.pool = Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81 74 self.pool.queue_total, 75 phases_names, 76 ) 77 self.pool.log_info("[BEGIN] %s" % (msg,)) 78 79 for phase in pipeline: 80 phase.start() --> 81 phase.stop() 82 83 self.pool.log_info("[DONE ] %s" % (msg,)) item = [, ] msg = 'Running pipeline (1/4). Phases: weaver, image_container' num = 1 phase = phases_names = 'weaver, image_container' pipeline = [, ] self = self._args = self._daemonic = False self._ident = 140009519703744 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-410' self._native_id = 1989551 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140009519703744 self.kill = True self.name = 'Thread-410' self.native_id = 1989551 self.pool = self.running = False self.traceparent = None Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67 60 61 self.pool.queue_get_lock.acquire() 62 self.pool.queue_processed += 1 63 num = self.pool.queue_processed 64 self.pool.queue_get_lock.release() 65 66 try: --> 67 self.process(item, num) 68 except: 69 self.failed = True 70 self.pool.exceptions.append(sys.exc_info()) 71 self.pool.kill() 72 73 def process(self, item, num): item = [, ] num = 1 self = self._args = self._daemonic = False self._ident = 140009519703744 self._initialized = True self._is_stopped = True self._kwargs = {} self._name = 'Thread-410' self._native_id = 1989551 self._started = self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'> self._target = None self._tstate_lock = None self.daemon = False self.failed = True self.get_timeout = 1 self.ident = 140009519703744 self.kill = True self.name = 'Thread-410' self.native_id = 1989551 self.pool = self.running = False self.traceparent = None Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719 712 713 def reraise(tp, value, tb=None): 714 try: 715 if value is None: 716 value = tp() 717 if value.__traceback__ is not tb: 718 raise value.with_traceback(tb) --> 719 raise value 720 finally: 721 value = None 722 tb = None 723 724 else: 725 def exec_(_code_, _globs_=None, _locs_=None): tb = None tp = value = None Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120 113 """ 114 for i in self.threads: 115 i.running = False 116 for i in self.threads: 117 i.join() 118 if self.exceptions: 119 exc_info = self.exceptions[0] --> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2]) 121 122 def kill(self): 123 for i in self.threads: 124 i.kill = True 125 i.running = False 126 exc_info = i = self = self._logger = self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'), self.queue_get_lock = self.queue_processed = 4 self.queue_put_lock = self.queue_total = 4 self.threads = [, , , Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59 52 53 self.pool.start() 54 55 def stop(self): 56 if self.finished: 57 return 58 if hasattr(self, "pool"): --> 59 self.pool.stop() 60 self.finished = True 61 self.compose.log_info("[DONE ] %s" % self.msg) 62 63 64 class PipelineThread(WorkerThread): 65 """ self = self._phases_schema = self.compose = self.finished = False self.msg = '---------- PHASE: WEAVER ----------' self.name = 'weaver' self.pool = Frame run_compose in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 524 517 ostree_installer_phase, 518 osbs_phase, 519 repoclosure_phase, 520 ) 521 extra_phase = pungi.phases.WeaverPhase(compose, extra_phase_schema) 522 523 extra_phase.start() --> 524 extra_phase.stop() 525 finally: 526 # wait for ostree container phase here too - it can happily run in parallel with 527 # all of the other stuff, but we must ensure it always gets stopped 528 ostree_container_phase.stop() 529 530 # now we do checksums as all images are done arch = 'src' buildinstall_phase = compose = compose_images_phase = compose_images_schema = config_copy_path = '/mnt/centos/composes/pungi/20260415-200120/logs/global/config-copy' config_dump_full = '/mnt/centos/composes/pungi/20260415-200120/logs/global/config-dump.global.log' config_file = '/home/composer/jenkins/workspace/stream10-gen2-compose@2/pungi-centos/centos/variables.conf' create_latest_link = True createiso_phase = createrepo_phase = errors = [] essentials_phase = essentials_schema = extra_isos_phase = extra_phase = extra_phase_schema = extrafiles_phase = f = <_io.TextIOWrapper name='/mnt/centos/composes/pungi/20260415-200120/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'> gather_phase = image_build_phase = image_checksum_phase = image_container_phase = imagebuilder_phase = init_phase = installed_pkgs_log = '/mnt/centos/composes/pungi/20260415-200120/logs/global/installed-pkgs.global.log' kiwibuild_phase = latest_link_components = -1 latest_link_status = None livemedia_phase = osbs_phase = osbuild_phase = ostree_container_phase = ostree_installer_phase = ostree_phase = phase = pkgset_phase = pungi = repoclosure_phase = test_phase = timestamp = '1776283999.150454' variant = Variant(id="SAPHANA", name="SAP Solutions", type="variant", parent=None) Frame main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 353 346 logger.error("Config validation failed with the error: %s" % error) 347 fail_to_start("Config validation failed", errors=errors) 348 sys.exit(1) 349 350 notifier.compose = compose 351 COMPOSE = compose 352 try: --> 353 run_compose( 354 compose, 355 create_latest_link=create_latest_link, 356 latest_link_status=latest_link_status, 357 latest_link_components=latest_link_components, 358 ) 359 except UnsignedPackagesError: Compose = PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'imagebuilder', 'image_checksum', 'image_container', 'init', 'kiwibuild', 'live_media', 'osbs', 'osbui abort = .abort at 0x7f5699c2cae0> ci = ci_path = '/mnt/centos/composes/pungi/20260415-200120/work/global/composeinfo-base.json' compose = compose_dir = '/mnt/centos/composes/pungi/20260415-200120' compose_type = 'test' conf = {'RELEASE_NAME': 'CentOS Stream', 'RELEASE_SHORT': 'CentOS-Stream', 'RELEASE_VERSION': '10', 'PKGSET_KOJI_TAG': 'c10s-compose', 'RUNROOT_CHANNEL': 'runroot', 'RUNROOT_TAG': 'c10s-build', 'RELEASE_VERS create_latest_link = True errors = [] fail_to_start = .fail_to_start at 0x7f5699c2c9a0> group = kobo = latest_link_components = -1 latest_link_status = None logger = notifier = opts = Namespace(target_dir=None, compose_dir='/mnt/centos/composes/pungi/20260415-200120', label=None, no_label=True, supported=False, old_composes=[], config='/home/composer/jenkins/workspace/stream10-gen2 parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=, conflict_handler='error', add_help=True) productmd = pungi = rv = None warning = 'WARNING: Unrecognized config option: EXTRA_BUILDINSTALL_SKIP.' warnings = ["WARNING: Config option release_is_layered was removed and has no effect; remove it. It's layered if there's configuration for base product.", "WARNING: Config option runroot was removed and has no e Frame cli_main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 658 651 signal.signal(signal.SIGINT, sigterm_handler) 652 signal.signal(signal.SIGTERM, sigterm_handler) 653 654 tracing.setup() 655 656 with tracing.span("run-compose"): 657 try: --> 658 main() 659 except (Exception, KeyboardInterrupt) as ex: 660 tracing.record_exception(ex) 661 if COMPOSE: 662 COMPOSE.log_error("Compose run failed: %s" % ex) 663 COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True)) 664 COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir) ex = RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details')