Skip to content

Commit 314c0fb

Browse files
authored
Detect Vast.ai interruptions during provisioning (#4037)
Detect spot interruptions and terminate the instance without waiting for provisioning timeout to elapse.
1 parent 5e39db1 commit 314c0fb

1 file changed

Lines changed: 10 additions & 0 deletions

File tree

  • src/dstack/_internal/core/backends/vastai

src/dstack/_internal/core/backends/vastai/compute.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,16 @@ def update_provisioning_data(
205205
and ": OCI runtime create failed:" in resp["status_msg"]
206206
):
207207
raise ProvisioningError(resp["status_msg"])
208+
if (
209+
resp.get("cur_state")
210+
== resp.get("intended_status")
211+
== resp.get("next_state")
212+
== "stopped"
213+
):
214+
# Can happen, among other cases, when a spot instance is outbid (interrupted)
215+
raise ProvisioningError(
216+
"Vast.ai reports current and intended instance state as `stopped`"
217+
)
208218

209219

210220
class VastAIOfferBackendData(CoreModel):

0 commit comments

Comments
 (0)