From e85ff3de70cfbe5dc246dea4845bdf77c982c5e3 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 7 Aug 2026 14:27:20 +0500 Subject: [PATCH 1/3] Report tried offers when new capacity provisioning fails A run that got a fleet but could not start on it terminated with no message at all. Collect what the offers loop tried, skipped and got back, and name the fleet, the number of offers and the backend errors. --- mkdocs/docs/guides/troubleshooting.md | 14 +- .../cli/services/configurators/run.py | 4 +- .../pipeline_tasks/jobs_submitted.py | 122 +++++++++++++++++- .../pipeline_tasks/test_submitted_jobs.py | 114 ++++++++++++++++ 4 files changed, 248 insertions(+), 6 deletions(-) diff --git a/mkdocs/docs/guides/troubleshooting.md b/mkdocs/docs/guides/troubleshooting.md index f3746a54d..3f7fe9f17 100644 --- a/mkdocs/docs/guides/troubleshooting.md +++ b/mkdocs/docs/guides/troubleshooting.md @@ -148,21 +148,29 @@ Alternatively, you can configure your own cloud accounts on the [project settings page](../concepts/projects.md#backends) or use [SSH fleets](../concepts/fleets.md#ssh-fleets). -### Provisioning fails +### Provisioning fails { #provisioning-fails } +[//]: # (NOTE: This section is referenced in the CLI. Do not change its URL.) In certain cases, running `dstack apply` may show instance offers, but then produce the following output: ```shell wet-mangust-1 provisioning completed (failed) -All provisioning attempts failed. This is likely due to cloud providers not having enough capacity. Check CLI and server logs for more details. +No offers +Failed to provision in fleet 'aws-main': tried 5 of 12 offers (attempt limit reached), all failed. +Errors: g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity; g5.xlarge in aws/eu-west-1: RequestLimitExceeded ``` +`dstack` only tries offers from the fleet it selected for the run, so the message names that +fleet, how many of its offers were tried, and what each attempt returned. Repeated errors are +reported once; every attempt is logged by the server, so check the [server logs](#server-logs) +for the full list. + #### Cause 1: Insufficient service quotas If some runs fail to provision, it may be due to an insufficient service quota. For cloud providers like AWS, GCP, Azure, and OCI, you often need to request an increased [service quota](protips.md#service-quotas) before you can use -specific instances. +specific instances. Quota errors returned by the backend are reported in the run's error message. ### Run starts but fails diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 335ca1dd8..793adf703 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -943,7 +943,9 @@ def print_finished_message(run: Run): console.print(str) if termination_reason_message: - console.print(f"[error]{termination_reason_message}[/error]") + # Backend errors reported in the message contain square brackets and numbers, + # which rich would otherwise parse as markup or repaint. + console.print(termination_reason_message, style="error", markup=False, highlight=False) if termination_reason: console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.") diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 74e1031c5..fdb8136f5 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -450,9 +450,39 @@ class _ExistingInstanceProvisioning: volume_attachment_result: _VolumeAttachmentResult +@dataclass +class _OfferAttemptError: + backend: str + region: str + instance: str + error: str + + +@dataclass +class _NewCapacityAttempts: + """ + What happened when the offers of the selected fleet were tried. + Used to explain why provisioning failed. + """ + + total: int + """Offers matching the run requirements at the time of provisioning.""" + tried: int + """Offers actually attempted. Lower than `total` if offers were skipped + or the attempt limit was reached.""" + skip_reasons: list[str] + """Why the offers that could not be attempted at all were skipped.""" + errors: list[_OfferAttemptError] + """Errors of the attempted offers.""" + limit_reached: bool + """Whether the loop stopped at `settings.MAX_OFFERS_TRIED` with offers left.""" + + @dataclass class _FailedNewCapacityProvisioning: placement_group_cleanup: Optional[_PlacementGroupCleanup] + message: Optional[str] = None + """Why the job could not be provisioned. `None` if the offers were never tried.""" @dataclass @@ -1449,6 +1479,7 @@ async def _process_new_capacity_provisioning( logger.debug("%s: provisioning failed", fmt(context.job_model)) return _TerminateSubmittedJobResult( reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + message=provision_new_capacity_result.message, locked_fleet_id=locked_fleet_id, placement_group_cleanup=provision_new_capacity_result.placement_group_cleanup, ) @@ -1475,6 +1506,55 @@ async def _process_new_capacity_provisioning( ) +_PROVISIONING_TROUBLESHOOTING_URL = ( + "https://dstack.ai/docs/guides/troubleshooting/#provisioning-fails" +) +_MAX_REPORTED_OFFER_ERRORS = 3 + + +def _get_new_capacity_failure_message( + fleet_name: str, + attempts: _NewCapacityAttempts, +) -> str: + if attempts.total == 0: + return ( + f"No offers matching the run requirements in fleet {fleet_name!r}." + f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + ) + if attempts.tried == 0: + return ( + f"None of the {attempts.total} offers in fleet {fleet_name!r} could be tried:" + f" {_format_reported_reasons(attempts.skip_reasons)}." + f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + ) + limit_reached = " (attempt limit reached)" if attempts.limit_reached else "" + message = ( + f"Failed to provision in fleet {fleet_name!r}:" + f" tried {attempts.tried} of {attempts.total} offers{limit_reached}, all failed." + ) + if attempts.errors: + message += f"\nErrors: {_format_offer_errors(attempts.errors)}." + return f"{message}\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + + +def _format_offer_errors(errors: list[_OfferAttemptError]) -> str: + # Offers commonly fail with the same error, so report every error once. + errors_by_message: dict[str, _OfferAttemptError] = {} + for error in errors: + errors_by_message.setdefault(error.error, error) + return _format_reported_reasons( + [f"{e.instance} in {e.backend}/{e.region}: {e.error}" for e in errors_by_message.values()] + ) + + +def _format_reported_reasons(reasons: list[str]) -> str: + unique_reasons = list(dict.fromkeys(reasons)) + reported = unique_reasons[:_MAX_REPORTED_OFFER_ERRORS] + if len(unique_reasons) > len(reported): + reported.append(f"and {len(unique_reasons) - len(reported)} more") + return "; ".join(reported) + + async def _apply_new_capacity_provisioning( session: AsyncSession, item: JobSubmittedPipelineItem, @@ -2233,10 +2313,14 @@ async def _provision_new_capacity( ) offers_iter = iter(offers) offers_tried = 0 + offers_taken = 0 + skip_reasons: list[str] = [] + offer_errors: list[_OfferAttemptError] = [] while offers_tried < settings.MAX_OFFERS_TRIED: backend_with_offer = next(offers_iter, None) if backend_with_offer is None: break + offers_taken += 1 backend, offer = backend_with_offer logger.debug( "%s: trying %s in %s/%s for $%0.4f per hour", @@ -2276,6 +2360,7 @@ async def _provision_new_capacity( compute=compute, ) if placement_group_model is None: + skip_reasons.append("no compatible placement group") continue if placement_group_model.id not in known_placement_group_ids: new_placement_group_models.append(placement_group_model) @@ -2334,6 +2419,7 @@ async def _provision_new_capacity( ) except SkipOffer as e: offers_tried -= 1 + skip_reasons.append(str(e) or "offer skipped") logger.info( "%s: %s launch in %s/%s skipped: %s", fmt(job_model), @@ -2344,6 +2430,7 @@ async def _provision_new_capacity( ) continue except BackendError as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) logger.warning( "%s: %s launch in %s/%s failed: %s", fmt(job_model), @@ -2353,7 +2440,8 @@ async def _provision_new_capacity( repr(e), ) continue - except Exception: + except Exception as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) logger.exception( "%s: got exception when launching %s in %s/%s", fmt(job_model), @@ -2363,12 +2451,42 @@ async def _provision_new_capacity( ) continue return _FailedNewCapacityProvisioning( + message=_get_new_capacity_failure_message( + fleet_name=fleet_model.name, + attempts=_NewCapacityAttempts( + total=len(offers), + tried=offers_tried, + skip_reasons=skip_reasons, + errors=offer_errors, + # Offers are left only if the attempt limit, not the offer list, ended the loop. + limit_reached=offers_taken < len(offers), + ), + ), placement_group_cleanup=_build_placement_group_cleanup( fleet_model=fleet_model, offers_tried=offers_tried, selected_placement_group_id=None, new_placement_group_models=new_placement_group_models, - ) + ), + ) + + +_MAX_OFFER_ERROR_LEN = 200 + + +def _get_offer_attempt_error( + offer: InstanceOfferWithAvailability, + error: Exception, +) -> _OfferAttemptError: + # Backend errors may be multiline and arbitrarily long since they often wrap cloud API errors. + message = " ".join(str(error).split()) or type(error).__name__ + if len(message) > _MAX_OFFER_ERROR_LEN: + message = message[:_MAX_OFFER_ERROR_LEN] + "..." + return _OfferAttemptError( + backend=offer.backend.value, + region=offer.region, + instance=offer.instance.name, + error=message, ) diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py index ee34ec5ad..7358cb573 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py @@ -44,7 +44,10 @@ JobSubmittedPipeline, JobSubmittedPipelineItem, JobSubmittedWorker, + _get_new_capacity_failure_message, _load_submitted_job_context, + _NewCapacityAttempts, + _OfferAttemptError, ) from dstack._internal.server.models import ( ComputeGroupModel, @@ -1968,6 +1971,39 @@ async def test_leaves_placeholder_for_terminating_pipeline_on_failed_new_capacit assert not placeholder.deleted assert placeholder.status == InstanceStatus.PENDING + async def test_reports_tried_offers_when_new_capacity_provisioning_fails( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=1) + fleet = await create_fleet(session=session, project=project, spec=fleet_spec) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job(session=session, run=run) + + offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us-east-1") + with patch("dstack._internal.server.services.backends.get_project_backends") as m: + backend_mock = Mock() + compute_mock = Mock(spec=ComputeMockSpec) + backend_mock.TYPE = BackendType.AWS + backend_mock.compute.return_value = compute_mock + m.return_value = [backend_mock] + compute_mock.get_offers.return_value = [offer] + compute_mock.run_job.side_effect = BackendError("InsufficientInstanceCapacity") + + # The first pass assigns the job to the fleet, the second one provisions. + await _process_job(session=session, worker=worker, job_model=job) + await _process_job(session=session, worker=worker, job_model=job) + + job = await _get_job(session, job.id) + assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + assert job.termination_reason_message is not None + assert f"Failed to provision in fleet '{fleet.name}'" in job.termination_reason_message + assert "tried 1 of 1 offers, all failed" in job.termination_reason_message + assert "us-east-1: InsufficientInstanceCapacity" in job.termination_reason_message + async def test_provisions_compute_group( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): @@ -2802,3 +2838,81 @@ async def test_loads_only_latest_submission(self, test_db, session: AsyncSession # Only the latest submission should be loaded. assert len(context.run_model.jobs) == 1 assert context.run_model.jobs[0].id == latest_job.id + + +class TestGetNewCapacityFailureMessage: + def _get_offer_attempt_error(self, region: str, error: str) -> _OfferAttemptError: + return _OfferAttemptError(backend="aws", region=region, instance="g5.xlarge", error=error) + + def test_reports_no_offers(self): + attempts = _NewCapacityAttempts( + total=0, tried=0, skip_reasons=[], errors=[], limit_reached=False + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "No offers matching the run requirements in fleet 'my-fleet'" in message + + def test_reports_skip_reasons_when_no_offer_was_tried(self): + attempts = _NewCapacityAttempts( + total=3, + tried=0, + skip_reasons=["no compatible placement group", "no compatible placement group"], + errors=[], + limit_reached=False, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "None of the 3 offers in fleet 'my-fleet' could be tried" in message + # Repeated reasons are reported once. + assert message.count("no compatible placement group") == 1 + + def test_reports_tried_offers_and_errors(self): + attempts = _NewCapacityAttempts( + total=12, + tried=5, + skip_reasons=[], + errors=[ + self._get_offer_attempt_error("us-east-1", "InsufficientInstanceCapacity"), + self._get_offer_attempt_error("us-west-2", "InsufficientInstanceCapacity"), + self._get_offer_attempt_error("eu-west-1", "RequestLimitExceeded"), + ], + limit_reached=True, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert ( + "Failed to provision in fleet 'my-fleet':" + " tried 5 of 12 offers (attempt limit reached), all failed." in message + ) + # The same error is reported once, for the first offer that hit it. + assert message.count("InsufficientInstanceCapacity") == 1 + assert "g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity" in message + assert "g5.xlarge in aws/eu-west-1: RequestLimitExceeded" in message + assert "us-west-2" not in message + + def test_does_not_report_the_attempt_limit_when_all_offers_were_tried(self): + attempts = _NewCapacityAttempts( + total=5, tried=5, skip_reasons=[], errors=[], limit_reached=False + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "tried 5 of 5 offers, all failed" in message + assert "attempt limit" not in message + + def test_truncates_reported_errors(self): + attempts = _NewCapacityAttempts( + total=5, + tried=5, + skip_reasons=[], + errors=[self._get_offer_attempt_error(f"region-{i}", f"error-{i}") for i in range(5)], + limit_reached=False, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "and 2 more" in message + assert "error-3" not in message From 14a24fc7188ef7806de4c02a72609202439820e2 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 7 Aug 2026 14:32:54 +0500 Subject: [PATCH 2/3] Rename the 'no offers' job status message to 'no capacity' --- mkdocs/docs/guides/troubleshooting.md | 4 ++-- src/dstack/_internal/cli/utils/run.py | 4 ++-- src/dstack/_internal/server/services/jobs/__init__.py | 2 +- src/tests/_internal/cli/utils/test_run.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs/docs/guides/troubleshooting.md b/mkdocs/docs/guides/troubleshooting.md index 3f7fe9f17..f74504b30 100644 --- a/mkdocs/docs/guides/troubleshooting.md +++ b/mkdocs/docs/guides/troubleshooting.md @@ -156,7 +156,7 @@ but then produce the following output: ```shell wet-mangust-1 provisioning completed (failed) -No offers +No capacity Failed to provision in fleet 'aws-main': tried 5 of 12 offers (attempt limit reached), all failed. Errors: g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity; g5.xlarge in aws/eu-west-1: RequestLimitExceeded ``` @@ -170,7 +170,7 @@ for the full list. If some runs fail to provision, it may be due to an insufficient service quota. For cloud providers like AWS, GCP, Azure, and OCI, you often need to request an increased [service quota](protips.md#service-quotas) before you can use -specific instances. Quota errors returned by the backend are reported in the run's error message. +specific instances. ### Run starts but fails diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index 6c27f2aa6..5464b4940 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -205,7 +205,7 @@ def _format_run_status(run) -> str: RunStatus.FAILED: "indian_red1", RunStatus.DONE: "grey", } - if status_text in ("no offers", "interrupted"): + if status_text in ("no capacity", "interrupted"): color = "gold1" elif status_text == "no fleets": color = "indian_red1" @@ -220,7 +220,7 @@ def _format_run_status(run) -> str: def _format_job_submission_status(job_submission: JobSubmission, verbose: bool) -> str: status_message = job_submission.status_message job_status = job_submission.status - if status_message in ("no offers", "interrupted"): + if status_message in ("no capacity", "interrupted"): color = "gold1" elif status_message == "no fleets": color = "indian_red1" diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index cff90a97e..728b0df55 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -597,7 +597,7 @@ def _get_job_status_message(job_model: JobModel) -> str: and "No matching fleet found" in job_model.termination_reason_message ): return "no fleets" - return "no offers" + return "no capacity" elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY: return "interrupted" else: diff --git a/src/tests/_internal/cli/utils/test_run.py b/src/tests/_internal/cli/utils/test_run.py index 6d7cbd90a..35bf28a43 100644 --- a/src/tests/_internal/cli/utils/test_run.py +++ b/src/tests/_internal/cli/utils/test_run.py @@ -255,7 +255,7 @@ async def test_simple_run(self, session: AsyncSession): JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, None, None, - "no offers", + "no capacity", "gold1", ), ( From 57087f79481e45718e4db5e6bf1909e14a142128 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 7 Aug 2026 14:55:36 +0500 Subject: [PATCH 3/3] Do not point at runner logs for a run that never started --- .../_internal/cli/services/configurators/run.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 793adf703..718b92e29 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -59,7 +59,14 @@ from dstack._internal.core.models.repos import RepoHeadWithCreds from dstack._internal.core.models.repos.base import Repo from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds -from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunPlan, RunSpec, RunStatus +from dstack._internal.core.models.runs import ( + JobStatus, + JobSubmission, + JobTerminationReason, + RunPlan, + RunSpec, + RunStatus, +) from dstack._internal.core.services.diff import diff_models from dstack._internal.core.services.repos import get_repo_creds_and_default_branch from dstack._internal.core.services.ssh.ports import PortUsedError @@ -947,7 +954,11 @@ def print_finished_message(run: Run): # which rich would otherwise parse as markup or repaint. console.print(termination_reason_message, style="error", markup=False, highlight=False) - if termination_reason: + if ( + termination_reason + # A run that never started has no runner logs to read. + and termination_reason != JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY.value + ): console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.")