# HELP queue_time_wait_seconds histogram of queue wait times
# TYPE queue_time_wait_seconds histogram
queue_time_wait_seconds_bucket{le="300"} 1976
queue_time_wait_seconds_bucket{le="3600"} 2601
queue_time_wait_seconds_bucket{le="+Inf"} 2755
queue_time_wait_seconds_sum 2719120
queue_time_wait_seconds_count 2755

# HELP test_run_seconds histogram of test execution time
# TYPE test_run_seconds histogram
test_run_seconds_bucket{le="3600"} 2716
test_run_seconds_bucket{le="7200"} 2754
test_run_seconds_bucket{le="+Inf"} 2755
test_run_seconds_sum 3019399
test_run_seconds_count 2755

# HELP top_failures_pct Most unstable tests in the last 7 days (more than 10% failure rate)
# TYPE top_failures_pct gauge
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse TestStorageHomeReuseDestructive.testHomeReuseFedoraEncrypted",context="fedora-rawhide-boot/anaconda-pr-7090"} 100.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testBasic",context="fedora-rawhide-boot/anaconda-pr-7090"} 100.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testExtendedPartition",context="fedora-rawhide-boot/efi-anaconda-pr-7090"} 100.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints-btrfs TestStorageMountPointsBtrfs.testBtrfsSubvolumes",context="fedora-rawhide-boot/efi-anaconda-pr-7090"} 100.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse TestStorageHomeReuse.testMultipleRoots",context="fedora-rawhide-boot/bios-anaconda-pr-7090"} 75.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testBasic",context="fedora-rawhide-boot/bios-anaconda-pr-7090"} 75.0
top_failures_pct{project="cockpit-project/cockpit-machines",test="/test/check-machines-multi-host-consoles TestMultiMachineVNC.testBasic",context="ubuntu-2604"} 60.0
top_failures_pct{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testSubAndInAndFail",context="rhel-10-3"} 50.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestKerberos.testNegotiate",context="fedora-44/networking"} 50.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-lvm2 TestStorageLvm2.testDegradation",context="fedora-44/firefox-storage"} 42.857142857142854
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-raid1 TestStorageRaid1.testMetadataAtEnd",context="centos-10/storage"} 41.666666666666664
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-networkmanager-settings TestNetworkingSettings.testNoConnectionSettings",context="ubuntu-2604/networking"} 38.46153846153846
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-raid1 TestStorageRaid1.testMetadataAtEnd",context="arch/storage"} 36.36363636363637
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-raid1 TestStorageRaid1.testMetadataAtEnd",context="rhel-10-3/storage"} 36.36363636363637
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-raid1 TestStorageRaid1.testRaidLevelOne",context="rhel-9-9/storage"} 36.36363636363637
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse TestStorageHomeReuse.testMultipleRoots",context="fedora-rawhide-boot/bios"} 35.714285714285715
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-used TestStorageUsed.testUsed",context="centos-9-bootc/storage"} 33.333333333333336
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-basic TestBasic.testSidebarNavigation",context="fedora-rawhide-boot/other"} 30.76923076923077
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-language TestLanguage.testKeyboardLayoutSelection",context="fedora-rawhide-boot/other"} 30.76923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-stratis TestStorageStratisReboot.testAtBoot",context="fedora-44/firefox-storage"} 30.76923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-networkmanager-mac TestNetworkingMAC.testMac",context="ubuntu-2604/networking"} 30.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-network TestNetworkDestructive.testCockpitConnectionsPreInstall",context="fedora-rawhide-boot/cockpit"} 28.571428571428573
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testEncryptedUnlockCockpit",context="fedora-rawhide-boot/storage"} 28.571428571428573
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-users TestAccounts.testCustomUID",context="debian-testing/other"} 26.666666666666668
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse-e2e TestStorageHomeReuse_E2E.testBasic",context="fedora-rawhide-boot/bios"} 25.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-metrics TestMetricsPackages.testBasic",context="centos-10/other"} 25.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-sosreport TestSOS.testBasic",context="fedora-43/other"} 25.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse TestStorageHomeReuseDestructive.testHomeReuseFedoraEncrypted",context="fedora-rawhide-boot/storage"} 23.076923076923077
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testBasic",context="fedora-rawhide-boot/storage"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-bots-api TestImageCustomize.testScriptRelativePath",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-bots-api TestImageCustomize.testUpload",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-client TestClient.testBeibootNoBridge",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-client TestClient.testBeibootWithBridge",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-connection TestReverseProxy.testNginxNoTLS",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-selinux TestSelinux.testTroubleshootAlerts",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-shell-multi-os TestRHEL8.test",context="debian-testing/other"} 23.076923076923077
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-lvm2 TestStorageLvm2.testDegradation",context="fedora-44/storage"} 22.22222222222222
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-lvm2 TestStorageLvm2.testDegradation",context="ubuntu-stable/storage"} 22.22222222222222
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-storage-raid1 TestStorageRaid1.testRaidLevelOne",context="rhel-10-3/storage"} 22.22222222222222
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testNotSupported",context="debian-trixie/networking"} 22.22222222222222
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPointsDestructive.testWindowsSingleDiskHomeReuse",context="fedora-rawhide-boot/storage"} 21.428571428571427
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-metrics TestCurrentMetrics.testCPU",context="debian-testing/other"} 21.428571428571427
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPoints.testBasic",context="fedora-rawhide-boot/bios"} 20.0
top_failures_pct{project="rhinstaller/anaconda-webui",test="test/check-storage-mountpoints TestStorageMountPointsDestructive.testMBRParttable",context="fedora-rawhide-boot/bios"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-networkmanager-mac TestNetworkingMAC.testBondMac",context="ubuntu-2604/networking"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestAutoUpdates.testBasic",context="fedora-44/devel"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestAutoUpdatesInstall.testUnsupported",context="rhel-8-10/ws-container-expensive"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestUpdates.testKpatch",context="rhel-8-10/ws-container-expensive"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-selinux TestSelinux.testTroubleshootAlerts",context="rhel-8-10/ws-container-other"} 20.0
top_failures_pct{project="cockpit-project/cockpit",test="test/verify/check-shell-menu TestMenu.testKeyboardNavigation",context="fedora-44/firefox-other"} 20.0

# HELP pr_retries histogram of how many retries it took to get PRs to green
# TYPE pr_retries histogram
pr_retries_bucket{le="0"} 260
pr_retries_bucket{le="1"} 346
pr_retries_bucket{le="2"} 362
pr_retries_bucket{le="3"} 366
pr_retries_bucket{le="4"} 368
pr_retries_bucket{le="5"} 370
pr_retries_bucket{le="6"} 371
pr_retries_bucket{le="+Inf"} 371
pr_retries_sum 371
pr_retries_count 371

# HELP merged_failures How many PRs were merged with non-successful tests
# TYPE merged_failures counter
merged_failures 8

# HELP top_slowest_tests The slowest tests in the last 7 days
# TYPE top_slowest_tests gauge
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-create TestMachinesCreate.testCreateDownloadAnOS"} 765
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-payload-dnf-e2e TestPayloadDNF_E2E.testDefaultClosestMirror"} 692
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-network-e2e TestNetwork_E2E.testConnectionsE2E"} 671
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-erase-all-e2e TestStorageEraseAll_E2E.testGuidedMultiEmptyAll"} 640
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-erase-all-e2e TestStorageEraseAll_E2E.testGuidedEncrypted"} 622
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-kickstart-automated TestKickstartAutomated.testBasic"} 594
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-cockpit-e2e TestStorageCockpitIntegration_E2E.testRAIDonPartitions"} 585
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-cockpit-e2e TestStorageCockpitIntegration_E2E.testSWAP"} 580
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-cockpit-e2e TestStorageCockpitIntegration_E2E.testRAID0Scenario_0"} 553
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-erase-all-e2e TestStorageEraseAll_E2E.testGuidedMultiSelect"} 547
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-progress TestInstallationProgress.testBasic"} 470
top_slowest_tests{project="rhinstaller/anaconda-webui",test="test/check-storage-home-reuse-e2e TestStorageHomeReuse_E2E.testBasic"} 465
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-storage-luks TestStorageNBDE.testRootReboot"} 435
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-storage-luks TestStorageNBDE.testRootReboot"} 375
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCPUSecurityMitigationsEnable"} 365
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 332
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testCryptoPolicies"} 314
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-info TestSystemInfo.testInconsistentCryptoPolicy"} 314
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 314
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 310
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-testlib TestRunTest.testRetry"} 304
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-kdump TestKdumpAnsible.testBasic"} 301
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-packagekit TestUpdates.testTracer"} 300
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 292
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 292
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-system-realms TestIPA.testUnqualifiedUsers"} 285
top_slowest_tests{project="cockpit-project/cockpit",test="test/verify/check-storage-stratis TestStorageStratisNBDE.testBasic"} 282
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicLibvirtUserUnprivileged"} 278
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-testlib TestRunTest.testRetry"} 261
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 249
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testOstree"} 243
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-system-info TestSystemInfo.testInconsistentCryptoPolicy"} 242
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testDeploymentManagement"} 238
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-storage-luks TestStorageNBDE.testBasic"} 238
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-system-realms TestIPA.testUnqualifiedUsers"} 222
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-system-info TestSystemInfo.testCryptoPolicies"} 215
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testSubAndInAndFail"} 215
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicSysadmU"} 214
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testInsights"} 208
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testInsights"} 204
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-system-realms TestIPA.testQualifiedUsers"} 200
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testInsights"} 191
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-storage-luks TestStorageLuksDestructive.testLuks1Slots"} 191
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-system-realms TestIPA.testClientCertAuthentication"} 188
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testDetachDisk"} 174
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-migrate TestMachinesMigration.testCopyStorageMigration"} 172
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-consoles TestMachinesConsoles.testAddSerial"} 168
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-kdump TestKdumpNFS.testBasic"} 166
top_slowest_tests{project="cockpit-project/cockpit/rhel-8",test="test/verify/check-metrics TestCurrentMetrics.testCPU"} 166
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 158
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-create TestMachinesCreate.testCreateBasicValidation"} 155
top_slowest_tests{project="cockpit-project/cockpit-machines",test="/test/check-machines-multi-host-consoles TestMultiMachineVNC.testBasic"} 155
top_slowest_tests{project="cockpit-project/cockpit-machines/rhel-8",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicAdminUser"} 155
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 151
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testRegister"} 147
top_slowest_tests{project="cockpit-project/cockpit-machines/rhel-8",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasicLibvirtUserUnprivileged"} 144
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 144
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegister"} 143
top_slowest_tests{project="cockpit-project/cockpit-machines/rhel-8",test="/test/check-machines-lifecycle TestMachinesLifecycle.testBasic"} 138
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 135
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testTranslations"} 130
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.29",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptionsPackages.testMissingPackages"} 129
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 128
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeCase.testPageStatus"} 127
top_slowest_tests{project="candlepin/subscription-manager",test="/cockpit-tests/submancockpit/test/check-subscriptions TestSubscriptions.testUnpriv"} 125
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeOCICase.testBasic"} 123
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testPodmanRestartEnabledUser"} 120
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testBasicSystem"} 118
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testTranslations"} 118
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testReconnectStability"} 116
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.29",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegister"} 115
top_slowest_tests{project="cockpit-project/cockpit-podman",test="/test/check-application TestApplication.testRunImageUser"} 112
top_slowest_tests{project="cockpit-project/subscription-manager-cockpit",test="/test/check-subscriptions TestSubscriptions.testUnpriv"} 112
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.29",test="/integration-tests/submancockpit/test/check-subscriptions TestSubscriptions.testRegisterWithKey"} 110
top_slowest_tests{project="cockpit-project/cockpit-machines/rhel-8",test="/test/check-machines-disks TestMachinesDisks.testAddDiskNFS"} 110
top_slowest_tests{project="candlepin/subscription-manager/subscription-manager-1.28",test="integration-tests/check-subscriptions TestSubscriptions.testRegister"} 107
top_slowest_tests{project="cockpit-project/cockpit-ostree",test="/test/check-ostree OstreeRestartCase.testRebase"} 107

# HELP s3_usage_bytes How much disk space an S3 store is using
# TYPE s3_usage_bytes gauge
s3_usage_bytes{store="https://cockpit-ci-images.s3.us-east-1.amazonaws.com/"} 103387494455

# HELP test_coverage Overall percentage of code covered by tests
# TYPE test_coverage gauge
test_coverage{project="cockpit-project/cockpit"} 85.9
test_coverage{project="cockpit-project/cockpit-ostree"} 73.2
test_coverage{project="cockpit-project/subscription-manager-cockpit"} 83.9