Increase LUCI build timeout to avoid fugu INFRA_FAILURE
One of our fugu devices failed and we have no replacement for now.
This means we have one fugu device handling two distinct builds,
which makes LUCI infra-fail because the builds are not started
in a timely manner. Increase the timeout to mitigate this.
When a new CL is submitted, this means that one of the fugu builds
will take ~11 hours to even start, let alone finish (hence the
current complains from LUCI), but let's tolerate it for now.
Test: ./validate.sh
Change-Id: Ia1d50b1a4c415310544ee7bee146c495e5b28b88
diff --git a/tools/luci/config/generated/cr-buildbucket.cfg b/tools/luci/config/generated/cr-buildbucket.cfg
index aa65641..8eddb51 100644
--- a/tools/luci/config/generated/cr-buildbucket.cfg
+++ b/tools/luci/config/generated/cr-buildbucket.cfg
@@ -25,8 +25,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -49,8 +49,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -73,8 +73,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -97,8 +97,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -121,8 +121,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -145,8 +145,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -169,8 +169,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -193,8 +193,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -217,8 +217,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -241,8 +241,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -265,8 +265,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -289,8 +289,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -313,8 +313,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -337,8 +337,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -361,8 +361,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -385,8 +385,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -409,8 +409,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -433,8 +433,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -457,8 +457,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -481,8 +481,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -505,8 +505,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -529,8 +529,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -553,8 +553,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -577,8 +577,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
@@ -601,8 +601,8 @@
cipd_version: "refs/heads/master"
properties_j: "builder_group:\"client.art\""
}
- execution_timeout_secs: 133200
- expiration_secs: 36000
+ execution_timeout_secs: 108000
+ expiration_secs: 61200
caches {
name: "art"
path: "art"
diff --git a/tools/luci/config/main.star b/tools/luci/config/main.star
index 748a3c1..79eb4df 100755
--- a/tools/luci/config/main.star
+++ b/tools/luci/config/main.star
@@ -166,10 +166,13 @@
},
service_account = "art-ci-builder@chops-service-accounts.iam.gserviceaccount.com",
- # We have a limited set of runners, so put the expiration time close to
- # the time it takes to run all steps on most builders.
- expiration_timeout = 10 * time.hour,
- execution_timeout = 37 * time.hour,
+ # Maximum delay between scheduling a build and the build actually starting.
+ # In a healthy state (enough free/idle devices), the delay is fairly small,
+ # but if enough devices are offline, this timeout will cause INFRA_FAILURE.
+ # Set the value reasonably high to prefer delayed builds over failing ones.
+ # NB: LUCI also enforces (expiration_timeout + execution_timeout <= 47).
+ expiration_timeout = 17 * time.hour,
+ execution_timeout = 30 * time.hour,
build_numbers = True,
properties = {
"builder_group": "client.art",