diff --git a/mkdocs/docs/guides/troubleshooting.md b/mkdocs/docs/guides/troubleshooting.md index f3746a54d..f74504b30 100644 --- a/mkdocs/docs/guides/troubleshooting.md +++ b/mkdocs/docs/guides/troubleshooting.md @@ -148,16 +148,24 @@ Alternatively, you can configure your own cloud accounts on the [project settings page](../concepts/projects.md#backends) or use [SSH fleets](../concepts/fleets.md#ssh-fleets). -### Provisioning fails +### Provisioning fails { #provisioning-fails } +[//]: # (NOTE: This section is referenced in the CLI. Do not change its URL.) In certain cases, running `dstack apply` may show instance offers, but then produce the following output: ```shell wet-mangust-1 provisioning completed (failed) -All provisioning attempts failed. This is likely due to cloud providers not having enough capacity. Check CLI and server logs for more details. +No capacity +Failed to provision in fleet 'aws-main': tried 5 of 12 offers (attempt limit reached), all failed. +Errors: g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity; g5.xlarge in aws/eu-west-1: RequestLimitExceeded ``` +`dstack` only tries offers from the fleet it selected for the run, so the message names that +fleet, how many of its offers were tried, and what each attempt returned. Repeated errors are +reported once; every attempt is logged by the server, so check the [server logs](#server-logs) +for the full list. + #### Cause 1: Insufficient service quotas If some runs fail to provision, it may be due to an insufficient service quota. For cloud providers like AWS, GCP, diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index 335ca1dd8..718b92e29 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -59,7 +59,14 @@ from dstack._internal.core.models.repos import RepoHeadWithCreds from dstack._internal.core.models.repos.base import Repo from dstack._internal.core.models.repos.remote import RemoteRepo, RemoteRepoCreds -from dstack._internal.core.models.runs import JobStatus, JobSubmission, RunPlan, RunSpec, RunStatus +from dstack._internal.core.models.runs import ( + JobStatus, + JobSubmission, + JobTerminationReason, + RunPlan, + RunSpec, + RunStatus, +) from dstack._internal.core.services.diff import diff_models from dstack._internal.core.services.repos import get_repo_creds_and_default_branch from dstack._internal.core.services.ssh.ports import PortUsedError @@ -943,9 +950,15 @@ def print_finished_message(run: Run): console.print(str) if termination_reason_message: - console.print(f"[error]{termination_reason_message}[/error]") - - if termination_reason: + # Backend errors reported in the message contain square brackets and numbers, + # which rich would otherwise parse as markup or repaint. + console.print(termination_reason_message, style="error", markup=False, highlight=False) + + if ( + termination_reason + # A run that never started has no runner logs to read. + and termination_reason != JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY.value + ): console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.") diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index 6c27f2aa6..5464b4940 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -205,7 +205,7 @@ def _format_run_status(run) -> str: RunStatus.FAILED: "indian_red1", RunStatus.DONE: "grey", } - if status_text in ("no offers", "interrupted"): + if status_text in ("no capacity", "interrupted"): color = "gold1" elif status_text == "no fleets": color = "indian_red1" @@ -220,7 +220,7 @@ def _format_run_status(run) -> str: def _format_job_submission_status(job_submission: JobSubmission, verbose: bool) -> str: status_message = job_submission.status_message job_status = job_submission.status - if status_message in ("no offers", "interrupted"): + if status_message in ("no capacity", "interrupted"): color = "gold1" elif status_message == "no fleets": color = "indian_red1" diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 74e1031c5..fdb8136f5 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -450,9 +450,39 @@ class _ExistingInstanceProvisioning: volume_attachment_result: _VolumeAttachmentResult +@dataclass +class _OfferAttemptError: + backend: str + region: str + instance: str + error: str + + +@dataclass +class _NewCapacityAttempts: + """ + What happened when the offers of the selected fleet were tried. + Used to explain why provisioning failed. + """ + + total: int + """Offers matching the run requirements at the time of provisioning.""" + tried: int + """Offers actually attempted. Lower than `total` if offers were skipped + or the attempt limit was reached.""" + skip_reasons: list[str] + """Why the offers that could not be attempted at all were skipped.""" + errors: list[_OfferAttemptError] + """Errors of the attempted offers.""" + limit_reached: bool + """Whether the loop stopped at `settings.MAX_OFFERS_TRIED` with offers left.""" + + @dataclass class _FailedNewCapacityProvisioning: placement_group_cleanup: Optional[_PlacementGroupCleanup] + message: Optional[str] = None + """Why the job could not be provisioned. `None` if the offers were never tried.""" @dataclass @@ -1449,6 +1479,7 @@ async def _process_new_capacity_provisioning( logger.debug("%s: provisioning failed", fmt(context.job_model)) return _TerminateSubmittedJobResult( reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + message=provision_new_capacity_result.message, locked_fleet_id=locked_fleet_id, placement_group_cleanup=provision_new_capacity_result.placement_group_cleanup, ) @@ -1475,6 +1506,55 @@ async def _process_new_capacity_provisioning( ) +_PROVISIONING_TROUBLESHOOTING_URL = ( + "https://dstack.ai/docs/guides/troubleshooting/#provisioning-fails" +) +_MAX_REPORTED_OFFER_ERRORS = 3 + + +def _get_new_capacity_failure_message( + fleet_name: str, + attempts: _NewCapacityAttempts, +) -> str: + if attempts.total == 0: + return ( + f"No offers matching the run requirements in fleet {fleet_name!r}." + f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + ) + if attempts.tried == 0: + return ( + f"None of the {attempts.total} offers in fleet {fleet_name!r} could be tried:" + f" {_format_reported_reasons(attempts.skip_reasons)}." + f"\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + ) + limit_reached = " (attempt limit reached)" if attempts.limit_reached else "" + message = ( + f"Failed to provision in fleet {fleet_name!r}:" + f" tried {attempts.tried} of {attempts.total} offers{limit_reached}, all failed." + ) + if attempts.errors: + message += f"\nErrors: {_format_offer_errors(attempts.errors)}." + return f"{message}\nSee {_PROVISIONING_TROUBLESHOOTING_URL}" + + +def _format_offer_errors(errors: list[_OfferAttemptError]) -> str: + # Offers commonly fail with the same error, so report every error once. + errors_by_message: dict[str, _OfferAttemptError] = {} + for error in errors: + errors_by_message.setdefault(error.error, error) + return _format_reported_reasons( + [f"{e.instance} in {e.backend}/{e.region}: {e.error}" for e in errors_by_message.values()] + ) + + +def _format_reported_reasons(reasons: list[str]) -> str: + unique_reasons = list(dict.fromkeys(reasons)) + reported = unique_reasons[:_MAX_REPORTED_OFFER_ERRORS] + if len(unique_reasons) > len(reported): + reported.append(f"and {len(unique_reasons) - len(reported)} more") + return "; ".join(reported) + + async def _apply_new_capacity_provisioning( session: AsyncSession, item: JobSubmittedPipelineItem, @@ -2233,10 +2313,14 @@ async def _provision_new_capacity( ) offers_iter = iter(offers) offers_tried = 0 + offers_taken = 0 + skip_reasons: list[str] = [] + offer_errors: list[_OfferAttemptError] = [] while offers_tried < settings.MAX_OFFERS_TRIED: backend_with_offer = next(offers_iter, None) if backend_with_offer is None: break + offers_taken += 1 backend, offer = backend_with_offer logger.debug( "%s: trying %s in %s/%s for $%0.4f per hour", @@ -2276,6 +2360,7 @@ async def _provision_new_capacity( compute=compute, ) if placement_group_model is None: + skip_reasons.append("no compatible placement group") continue if placement_group_model.id not in known_placement_group_ids: new_placement_group_models.append(placement_group_model) @@ -2334,6 +2419,7 @@ async def _provision_new_capacity( ) except SkipOffer as e: offers_tried -= 1 + skip_reasons.append(str(e) or "offer skipped") logger.info( "%s: %s launch in %s/%s skipped: %s", fmt(job_model), @@ -2344,6 +2430,7 @@ async def _provision_new_capacity( ) continue except BackendError as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) logger.warning( "%s: %s launch in %s/%s failed: %s", fmt(job_model), @@ -2353,7 +2440,8 @@ async def _provision_new_capacity( repr(e), ) continue - except Exception: + except Exception as e: + offer_errors.append(_get_offer_attempt_error(offer=offer, error=e)) logger.exception( "%s: got exception when launching %s in %s/%s", fmt(job_model), @@ -2363,12 +2451,42 @@ async def _provision_new_capacity( ) continue return _FailedNewCapacityProvisioning( + message=_get_new_capacity_failure_message( + fleet_name=fleet_model.name, + attempts=_NewCapacityAttempts( + total=len(offers), + tried=offers_tried, + skip_reasons=skip_reasons, + errors=offer_errors, + # Offers are left only if the attempt limit, not the offer list, ended the loop. + limit_reached=offers_taken < len(offers), + ), + ), placement_group_cleanup=_build_placement_group_cleanup( fleet_model=fleet_model, offers_tried=offers_tried, selected_placement_group_id=None, new_placement_group_models=new_placement_group_models, - ) + ), + ) + + +_MAX_OFFER_ERROR_LEN = 200 + + +def _get_offer_attempt_error( + offer: InstanceOfferWithAvailability, + error: Exception, +) -> _OfferAttemptError: + # Backend errors may be multiline and arbitrarily long since they often wrap cloud API errors. + message = " ".join(str(error).split()) or type(error).__name__ + if len(message) > _MAX_OFFER_ERROR_LEN: + message = message[:_MAX_OFFER_ERROR_LEN] + "..." + return _OfferAttemptError( + backend=offer.backend.value, + region=offer.region, + instance=offer.instance.name, + error=message, ) diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index cff90a97e..728b0df55 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -597,7 +597,7 @@ def _get_job_status_message(job_model: JobModel) -> str: and "No matching fleet found" in job_model.termination_reason_message ): return "no fleets" - return "no offers" + return "no capacity" elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY: return "interrupted" else: diff --git a/src/tests/_internal/cli/utils/test_run.py b/src/tests/_internal/cli/utils/test_run.py index 6d7cbd90a..35bf28a43 100644 --- a/src/tests/_internal/cli/utils/test_run.py +++ b/src/tests/_internal/cli/utils/test_run.py @@ -255,7 +255,7 @@ async def test_simple_run(self, session: AsyncSession): JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, None, None, - "no offers", + "no capacity", "gold1", ), ( diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py index ee34ec5ad..7358cb573 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py @@ -44,7 +44,10 @@ JobSubmittedPipeline, JobSubmittedPipelineItem, JobSubmittedWorker, + _get_new_capacity_failure_message, _load_submitted_job_context, + _NewCapacityAttempts, + _OfferAttemptError, ) from dstack._internal.server.models import ( ComputeGroupModel, @@ -1968,6 +1971,39 @@ async def test_leaves_placeholder_for_terminating_pipeline_on_failed_new_capacit assert not placeholder.deleted assert placeholder.status == InstanceStatus.PENDING + async def test_reports_tried_offers_when_new_capacity_provisioning_fails( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + fleet_spec = get_fleet_spec() + fleet_spec.configuration.nodes = FleetNodesSpec(min=0, target=0, max=1) + fleet = await create_fleet(session=session, project=project, spec=fleet_spec) + run = await create_run(session=session, project=project, repo=repo, user=user) + job = await create_job(session=session, run=run) + + offer = get_instance_offer_with_availability(backend=BackendType.AWS, region="us-east-1") + with patch("dstack._internal.server.services.backends.get_project_backends") as m: + backend_mock = Mock() + compute_mock = Mock(spec=ComputeMockSpec) + backend_mock.TYPE = BackendType.AWS + backend_mock.compute.return_value = compute_mock + m.return_value = [backend_mock] + compute_mock.get_offers.return_value = [offer] + compute_mock.run_job.side_effect = BackendError("InsufficientInstanceCapacity") + + # The first pass assigns the job to the fleet, the second one provisions. + await _process_job(session=session, worker=worker, job_model=job) + await _process_job(session=session, worker=worker, job_model=job) + + job = await _get_job(session, job.id) + assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + assert job.termination_reason_message is not None + assert f"Failed to provision in fleet '{fleet.name}'" in job.termination_reason_message + assert "tried 1 of 1 offers, all failed" in job.termination_reason_message + assert "us-east-1: InsufficientInstanceCapacity" in job.termination_reason_message + async def test_provisions_compute_group( self, test_db, session: AsyncSession, worker: JobSubmittedWorker ): @@ -2802,3 +2838,81 @@ async def test_loads_only_latest_submission(self, test_db, session: AsyncSession # Only the latest submission should be loaded. assert len(context.run_model.jobs) == 1 assert context.run_model.jobs[0].id == latest_job.id + + +class TestGetNewCapacityFailureMessage: + def _get_offer_attempt_error(self, region: str, error: str) -> _OfferAttemptError: + return _OfferAttemptError(backend="aws", region=region, instance="g5.xlarge", error=error) + + def test_reports_no_offers(self): + attempts = _NewCapacityAttempts( + total=0, tried=0, skip_reasons=[], errors=[], limit_reached=False + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "No offers matching the run requirements in fleet 'my-fleet'" in message + + def test_reports_skip_reasons_when_no_offer_was_tried(self): + attempts = _NewCapacityAttempts( + total=3, + tried=0, + skip_reasons=["no compatible placement group", "no compatible placement group"], + errors=[], + limit_reached=False, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "None of the 3 offers in fleet 'my-fleet' could be tried" in message + # Repeated reasons are reported once. + assert message.count("no compatible placement group") == 1 + + def test_reports_tried_offers_and_errors(self): + attempts = _NewCapacityAttempts( + total=12, + tried=5, + skip_reasons=[], + errors=[ + self._get_offer_attempt_error("us-east-1", "InsufficientInstanceCapacity"), + self._get_offer_attempt_error("us-west-2", "InsufficientInstanceCapacity"), + self._get_offer_attempt_error("eu-west-1", "RequestLimitExceeded"), + ], + limit_reached=True, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert ( + "Failed to provision in fleet 'my-fleet':" + " tried 5 of 12 offers (attempt limit reached), all failed." in message + ) + # The same error is reported once, for the first offer that hit it. + assert message.count("InsufficientInstanceCapacity") == 1 + assert "g5.xlarge in aws/us-east-1: InsufficientInstanceCapacity" in message + assert "g5.xlarge in aws/eu-west-1: RequestLimitExceeded" in message + assert "us-west-2" not in message + + def test_does_not_report_the_attempt_limit_when_all_offers_were_tried(self): + attempts = _NewCapacityAttempts( + total=5, tried=5, skip_reasons=[], errors=[], limit_reached=False + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "tried 5 of 5 offers, all failed" in message + assert "attempt limit" not in message + + def test_truncates_reported_errors(self): + attempts = _NewCapacityAttempts( + total=5, + tried=5, + skip_reasons=[], + errors=[self._get_offer_attempt_error(f"region-{i}", f"error-{i}") for i in range(5)], + limit_reached=False, + ) + + message = _get_new_capacity_failure_message(fleet_name="my-fleet", attempts=attempts) + + assert "and 2 more" in message + assert "error-3" not in message