Traceback (most recent call last):
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 658, in cli_main
main()
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 353, in main
run_compose(
File "/usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py", line 524, in run_compose
extra_phase.stop()
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop
self.pool.stop()
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
six.reraise(exc_info[0], exc_info[1], exc_info[2])
File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
self.process(item, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process
phase.stop()
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 59, in stop
self.pool.stop()
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
six.reraise(exc_info[0], exc_info[1], exc_info[2])
File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
self.process(item, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/weaver.py", line 81, in process
phase.stop()
File "/usr/lib/python3.12/site-packages/pungi/phases/base.py", line 111, in stop
self.pool.stop()
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 120, in stop
six.reraise(exc_info[0], exc_info[1], exc_info[2])
File "/usr/lib/python3.12/site-packages/six.py", line 719, in reraise
raise value
File "/usr/lib/python3.12/site-packages/kobo/threads.py", line 67, in run
self.process(item, num)
File "/usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py", line 134, in process
self.worker(compose, variant, config, arches, generics, repo)
File "/usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py", line 178, in worker
raise RuntimeError(
RuntimeError: imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details
Frame worker in /usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py at line 178
171 # Wait for it to finish and capture the output into log file.
172 log_dir = os.path.join(compose.paths.log.topdir(), "imageBuilderBuild")
173 util.makedirs(log_dir)
174 log_file = os.path.join(
175 log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
176 )
177 if koji.watch_task(task_id, log_file) != 0:
--> 178 raise RuntimeError(
179 "imageBuilderBuild task failed: %s. See %s for details"
180 % (task_id, log_file)
181 )
182
183 # Refresh koji session which may have timed out while the task was
184 # running. Watching is done via a subprocess, so the session is
arches = ['aarch64', 'ppc64le', 's390x', 'x86_64']
compose =
config = {'name': 'CentOS-Stream-GenericCloud', 'version': '10', 'distro': 'centos-stream-10', 'types': ['qcow2'], 'target': 'centos-10', 'arches': ['aarch64', 'ppc64le', 's390x', 'x86_64'], 'repos': ['BaseOS'
generics = {'release': '20260415.t.0', 'target': 'centos-10', 'types': ['qcow2'], 'seed': None, 'scratch': None, 'version': '10'}
koji =
log_dir = '/mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild'
log_file = '/mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log'
msg = 'imageBuilderBuild task for variant BaseOS'
opts = {'repos': ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os'], '
repo = ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os']
self =
self._args =
self._daemonic = False
self._ident = 140008839444160
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-425'
self._native_id = 1989567
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failable_arches = []
self.failed = True
self.get_timeout = 1
self.ident = 140008839444160
self.kill = True
self.name = 'Thread-425'
self.native_id = 1989567
self.num = 1
self.pool =
self.running = False
self.traceparent = None
task_id = 6249513
variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None)
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/imagebuilder.py at line 134
127 compose,
128 can_task_fail,
129 variant,
130 "*",
131 "imageBuilderBuild",
132 logger=self.pool._logger,
133 ):
--> 134 self.worker(compose, variant, config, arches, generics, repo)
135
136 def worker(self, compose, variant, config, arches, generics, repo):
137 msg = "imageBuilderBuild task for variant %s" % variant.uid
138 self.pool.log_info("[BEGIN] %s" % msg)
139 koji = kojiwrapper.KojiWrapper(compose)
140 koji.login()
arches = ['aarch64', 'ppc64le', 's390x', 'x86_64']
can_task_fail = False
compose =
config = {'name': 'CentOS-Stream-GenericCloud', 'version': '10', 'distro': 'centos-stream-10', 'types': ['qcow2'], 'target': 'centos-10', 'arches': ['aarch64', 'ppc64le', 's390x', 'x86_64'], 'repos': ['BaseOS'
failable_arches = []
generics = {'release': '20260415.t.0', 'target': 'centos-10', 'types': ['qcow2'], 'seed': None, 'scratch': None, 'version': '10'}
item =
num = 1
repo = ['http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/BaseOS/$arch/os', 'http://composefiles.stream.rdu2.redhat.com/pungi/20260415-200120/compose/AppStream/$arch/os']
self =
self._args =
self._daemonic = False
self._ident = 140008839444160
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-425'
self._native_id = 1989567
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failable_arches = []
self.failed = True
self.get_timeout = 1
self.ident = 140008839444160
self.kill = True
self.name = 'Thread-425'
self.native_id = 1989567
self.num = 1
self.pool =
self.running = False
self.traceparent = None
variant = Variant(id="BaseOS", name="BaseOS", type="variant", parent=None)
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
60
61 self.pool.queue_get_lock.acquire()
62 self.pool.queue_processed += 1
63 num = self.pool.queue_processed
64 self.pool.queue_get_lock.release()
65
66 try:
--> 67 self.process(item, num)
68 except:
69 self.failed = True
70 self.pool.exceptions.append(sys.exc_info())
71 self.pool.kill()
72
73 def process(self, item, num):
item =
num = 1
self =
self._args =
self._daemonic = False
self._ident = 140008839444160
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-425'
self._native_id = 1989567
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failable_arches = []
self.failed = True
self.get_timeout = 1
self.ident = 140008839444160
self.kill = True
self.name = 'Thread-425'
self.native_id = 1989567
self.num = 1
self.pool =
self.running = False
self.traceparent = None
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
712
713 def reraise(tp, value, tb=None):
714 try:
715 if value is None:
716 value = tp()
717 if value.__traceback__ is not tb:
718 raise value.with_traceback(tb)
--> 719 raise value
720 finally:
721 value = None
722 tb = None
723
724 else:
725 def exec_(_code_, _globs_=None, _locs_=None):
tb = None
tp =
value = None
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
113 """
114 for i in self.threads:
115 i.running = False
116 for i in self.threads:
117 i.join()
118 if self.exceptions:
119 exc_info = self.exceptions[0]
--> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2])
121
122 def kill(self):
123 for i in self.threads:
124 i.kill = True
125 i.running = False
126
exc_info =
i =
self =
self._logger =
self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'),
self.queue_get_lock =
self.queue_processed = 2
self.queue_put_lock =
self.queue_total = 2
self.threads = [, ]
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/base.py at line 111
104 "Note that variants can be excluded in configuration file"
105 )
106
107 def stop(self):
108 if self.finished:
109 return
110 if hasattr(self, "pool"):
--> 111 self.pool.stop()
112 self.finished = True
113 self.compose.log_info("[DONE ] %s" % self.msg)
114
115 if hasattr(self, "_start_time"):
116 self.compose.log_info(
117 "PHASE %s took %d seconds"
self =
self._skipped = False
self._start_time = 1776283999.1601636
self.compose =
self.finished = False
self.logger =
self.msg = '---------- PHASE: IMAGEBUILDER ----------'
self.name = 'imagebuilder'
self.pool =
self.used_patterns = {'^BaseOS$'}
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81
74 self.pool.queue_total,
75 phases_names,
76 )
77 self.pool.log_info("[BEGIN] %s" % (msg,))
78
79 for phase in pipeline:
80 phase.start()
--> 81 phase.stop()
82
83 self.pool.log_info("[DONE ] %s" % (msg,))
item = []
msg = 'Running pipeline (7/7). Phases: imagebuilder'
num = 7
phase =
phases_names = 'imagebuilder'
pipeline = []
self =
self._args =
self._daemonic = False
self._ident = 140008475322048
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-420'
self._native_id = 1989565
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140008475322048
self.kill = True
self.name = 'Thread-420'
self.native_id = 1989565
self.pool =
self.running = False
self.traceparent = None
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
60
61 self.pool.queue_get_lock.acquire()
62 self.pool.queue_processed += 1
63 num = self.pool.queue_processed
64 self.pool.queue_get_lock.release()
65
66 try:
--> 67 self.process(item, num)
68 except:
69 self.failed = True
70 self.pool.exceptions.append(sys.exc_info())
71 self.pool.kill()
72
73 def process(self, item, num):
item = []
num = 7
self =
self._args =
self._daemonic = False
self._ident = 140008475322048
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-420'
self._native_id = 1989565
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140008475322048
self.kill = True
self.name = 'Thread-420'
self.native_id = 1989565
self.pool =
self.running = False
self.traceparent = None
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
712
713 def reraise(tp, value, tb=None):
714 try:
715 if value is None:
716 value = tp()
717 if value.__traceback__ is not tb:
718 raise value.with_traceback(tb)
--> 719 raise value
720 finally:
721 value = None
722 tb = None
723
724 else:
725 def exec_(_code_, _globs_=None, _locs_=None):
tb = None
tp =
value = None
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
113 """
114 for i in self.threads:
115 i.running = False
116 for i in self.threads:
117 i.join()
118 if self.exceptions:
119 exc_info = self.exceptions[0]
--> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2])
121
122 def kill(self):
123 for i in self.threads:
124 i.kill = True
125 i.running = False
126
exc_info =
i =
self =
self._logger =
self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'),
self.queue_get_lock =
self.queue_processed = 7
self.queue_put_lock =
self.queue_total = 7
self.threads = [, , ,
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59
52
53 self.pool.start()
54
55 def stop(self):
56 if self.finished:
57 return
58 if hasattr(self, "pool"):
--> 59 self.pool.stop()
60 self.finished = True
61 self.compose.log_info("[DONE ] %s" % self.msg)
62
63
64 class PipelineThread(WorkerThread):
65 """
self =
self._phases_schema =
self.compose =
self.finished = False
self.msg = '---------- PHASE: WEAVER ----------'
self.name = 'weaver'
self.pool =
Frame process in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 81
74 self.pool.queue_total,
75 phases_names,
76 )
77 self.pool.log_info("[BEGIN] %s" % (msg,))
78
79 for phase in pipeline:
80 phase.start()
--> 81 phase.stop()
82
83 self.pool.log_info("[DONE ] %s" % (msg,))
item = [, ]
msg = 'Running pipeline (1/4). Phases: weaver, image_container'
num = 1
phase =
phases_names = 'weaver, image_container'
pipeline = [, ]
self =
self._args =
self._daemonic = False
self._ident = 140009519703744
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-410'
self._native_id = 1989551
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140009519703744
self.kill = True
self.name = 'Thread-410'
self.native_id = 1989551
self.pool =
self.running = False
self.traceparent = None
Frame run in /usr/lib/python3.12/site-packages/kobo/threads.py at line 67
60
61 self.pool.queue_get_lock.acquire()
62 self.pool.queue_processed += 1
63 num = self.pool.queue_processed
64 self.pool.queue_get_lock.release()
65
66 try:
--> 67 self.process(item, num)
68 except:
69 self.failed = True
70 self.pool.exceptions.append(sys.exc_info())
71 self.pool.kill()
72
73 def process(self, item, num):
item = [, ]
num = 1
self =
self._args =
self._daemonic = False
self._ident = 140009519703744
self._initialized = True
self._is_stopped = True
self._kwargs = {}
self._name = 'Thread-410'
self._native_id = 1989551
self._started =
self._stderr = <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>
self._target = None
self._tstate_lock = None
self.daemon = False
self.failed = True
self.get_timeout = 1
self.ident = 140009519703744
self.kill = True
self.name = 'Thread-410'
self.native_id = 1989551
self.pool =
self.running = False
self.traceparent = None
Frame reraise in /usr/lib/python3.12/site-packages/six.py at line 719
712
713 def reraise(tp, value, tb=None):
714 try:
715 if value is None:
716 value = tp()
717 if value.__traceback__ is not tb:
718 raise value.with_traceback(tb)
--> 719 raise value
720 finally:
721 value = None
722 tb = None
723
724 else:
725 def exec_(_code_, _globs_=None, _locs_=None):
tb = None
tp =
value = None
Frame stop in /usr/lib/python3.12/site-packages/kobo/threads.py at line 120
113 """
114 for i in self.threads:
115 i.running = False
116 for i in self.threads:
117 i.join()
118 if self.exceptions:
119 exc_info = self.exceptions[0]
--> 120 six.reraise(exc_info[0], exc_info[1], exc_info[2])
121
122 def kill(self):
123 for i in self.threads:
124 i.kill = True
125 i.running = False
126
exc_info =
i =
self =
self._logger =
self.exceptions = [(, RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details'),
self.queue_get_lock =
self.queue_processed = 4
self.queue_put_lock =
self.queue_total = 4
self.threads = [, , ,
Frame stop in /usr/lib/python3.12/site-packages/pungi/phases/weaver.py at line 59
52
53 self.pool.start()
54
55 def stop(self):
56 if self.finished:
57 return
58 if hasattr(self, "pool"):
--> 59 self.pool.stop()
60 self.finished = True
61 self.compose.log_info("[DONE ] %s" % self.msg)
62
63
64 class PipelineThread(WorkerThread):
65 """
self =
self._phases_schema =
self.compose =
self.finished = False
self.msg = '---------- PHASE: WEAVER ----------'
self.name = 'weaver'
self.pool =
Frame run_compose in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 524
517 ostree_installer_phase,
518 osbs_phase,
519 repoclosure_phase,
520 )
521 extra_phase = pungi.phases.WeaverPhase(compose, extra_phase_schema)
522
523 extra_phase.start()
--> 524 extra_phase.stop()
525 finally:
526 # wait for ostree container phase here too - it can happily run in parallel with
527 # all of the other stuff, but we must ensure it always gets stopped
528 ostree_container_phase.stop()
529
530 # now we do checksums as all images are done
arch = 'src'
buildinstall_phase =
compose =
compose_images_phase =
compose_images_schema =
config_copy_path = '/mnt/centos/composes/pungi/20260415-200120/logs/global/config-copy'
config_dump_full = '/mnt/centos/composes/pungi/20260415-200120/logs/global/config-dump.global.log'
config_file = '/home/composer/jenkins/workspace/stream10-gen2-compose@2/pungi-centos/centos/variables.conf'
create_latest_link = True
createiso_phase =
createrepo_phase =
errors = []
essentials_phase =
essentials_schema =
extra_isos_phase =
extra_phase =
extra_phase_schema =
extrafiles_phase =
f = <_io.TextIOWrapper name='/mnt/centos/composes/pungi/20260415-200120/logs/global/config-dump.global.log' mode='w' encoding='UTF-8'>
gather_phase =
image_build_phase =
image_checksum_phase =
image_container_phase =
imagebuilder_phase =
init_phase =
installed_pkgs_log = '/mnt/centos/composes/pungi/20260415-200120/logs/global/installed-pkgs.global.log'
kiwibuild_phase =
latest_link_components = -1
latest_link_status = None
livemedia_phase =
osbs_phase =
osbuild_phase =
ostree_container_phase =
ostree_installer_phase =
ostree_phase =
phase =
pkgset_phase =
pungi =
repoclosure_phase =
test_phase =
timestamp = '1776283999.150454'
variant = Variant(id="SAPHANA", name="SAP Solutions", type="variant", parent=None)
Frame main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 353
346 logger.error("Config validation failed with the error: %s" % error)
347 fail_to_start("Config validation failed", errors=errors)
348 sys.exit(1)
349
350 notifier.compose = compose
351 COMPOSE = compose
352 try:
--> 353 run_compose(
354 compose,
355 create_latest_link=create_latest_link,
356 latest_link_status=latest_link_status,
357 latest_link_components=latest_link_components,
358 )
359 except UnsignedPackagesError:
Compose =
PHASES_NAMES_MODIFIED = ['buildinstall', 'createiso', 'createrepo', 'extra_files', 'extra_isos', 'gather', 'image_build', 'imagebuilder', 'image_checksum', 'image_container', 'init', 'kiwibuild', 'live_media', 'osbs', 'osbui
abort = .abort at 0x7f5699c2cae0>
ci =
ci_path = '/mnt/centos/composes/pungi/20260415-200120/work/global/composeinfo-base.json'
compose =
compose_dir = '/mnt/centos/composes/pungi/20260415-200120'
compose_type = 'test'
conf = {'RELEASE_NAME': 'CentOS Stream', 'RELEASE_SHORT': 'CentOS-Stream', 'RELEASE_VERSION': '10', 'PKGSET_KOJI_TAG': 'c10s-compose', 'RUNROOT_CHANNEL': 'runroot', 'RUNROOT_TAG': 'c10s-build', 'RELEASE_VERS
create_latest_link = True
errors = []
fail_to_start = .fail_to_start at 0x7f5699c2c9a0>
group =
kobo =
latest_link_components = -1
latest_link_status = None
logger =
notifier =
opts = Namespace(target_dir=None, compose_dir='/mnt/centos/composes/pungi/20260415-200120', label=None, no_label=True, supported=False, old_composes=[], config='/home/composer/jenkins/workspace/stream10-gen2
parser = ArgumentParser(prog='pungi-koji', usage=None, description=None, formatter_class=, conflict_handler='error', add_help=True)
productmd =
pungi =
rv = None
warning = 'WARNING: Unrecognized config option: EXTRA_BUILDINSTALL_SKIP.'
warnings = ["WARNING: Config option release_is_layered was removed and has no effect; remove it. It's layered if there's configuration for base product.", "WARNING: Config option runroot was removed and has no e
Frame cli_main in /usr/lib/python3.12/site-packages/pungi/scripts/pungi_koji.py at line 658
651 signal.signal(signal.SIGINT, sigterm_handler)
652 signal.signal(signal.SIGTERM, sigterm_handler)
653
654 tracing.setup()
655
656 with tracing.span("run-compose"):
657 try:
--> 658 main()
659 except (Exception, KeyboardInterrupt) as ex:
660 tracing.record_exception(ex)
661 if COMPOSE:
662 COMPOSE.log_error("Compose run failed: %s" % ex)
663 COMPOSE.traceback(show_locals=getattr(ex, "show_locals", True))
664 COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
ex = RuntimeError('imageBuilderBuild task failed: 6249513. See /mnt/centos/composes/pungi/20260415-200120/logs/global/imageBuilderBuild/BaseOS-1-watch-task.log for details')