diff --git a/.agents/scripts/prepare-marimo-example.sh b/.agents/scripts/prepare-marimo-example.sh deleted file mode 100755 index 459ffae0..00000000 --- a/.agents/scripts/prepare-marimo-example.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -usage() { - echo "usage: $0 notebook.ipynb --name example-name [--force] [--fail-on-check]" >&2 -} - -script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" -repo_root="$(cd -- "$script_dir/../.." && pwd)" - -input="${1:-}" -if [[ -z "$input" || "$input" == "-h" || "$input" == "--help" ]]; then - usage - if [[ -z "$input" ]]; then - exit 1 - fi - exit 0 -fi -shift - -name="" -force=0 -fail_on_check=0 -while (($#)); do - case "$1" in - --name) - shift - (($#)) || { - echo "missing value for --name" >&2 - exit 1 - } - name="$1" - ;; - --force) - force=1 - ;; - --fail-on-check) - fail_on_check=1 - ;; - *) - echo "unknown argument: $1" >&2 - exit 1 - ;; - esac - shift -done - -[[ -n "$name" ]] || { - usage - exit 1 -} - -[[ "$input" == *.ipynb ]] || { - echo "input must be a .ipynb file: $input" >&2 - exit 1 -} - -[[ -f "$input" ]] || { - echo "input file does not exist: $input" >&2 - exit 1 -} - -input_dir="$(cd -- "$(dirname -- "$input")" && pwd)" -input="$input_dir/$(basename -- "$input")" - -slug="$name" -[[ "$slug" =~ ^[A-Za-z0-9][A-Za-z0-9_-]*$ ]] || { - echo "--name must be a slug like 'mnist-registry' or 'mnist_registry' (no paths, dots, or spaces): $slug" >&2 - exit 1 -} - -module_name="${slug//-/_}" -target_dir="examples/marimo/$slug" -target_py="$target_dir/$module_name.py" -debug_dir="$target_dir/.conversion" -report="$debug_dir/conversion-report.md" -check_output="$debug_dir/marimo-check.txt" - -cd "$repo_root" -if [[ -e "$target_py" && "$force" -eq 0 ]]; then - echo "target notebook already exists: $target_py" >&2 - echo "pass --force to overwrite it" >&2 - exit 1 -fi -mkdir -p "$target_dir" "$debug_dir" - -uvx marimo convert "$input" -o "$target_py" - -check_status=0 -uvx marimo check "$target_py" > "$check_output" 2>&1 || check_status=$? - -cat > "$report" <&2 -fi - -if ((fail_on_check)); then - exit "$check_status" -fi diff --git a/.agents/skills/README.md b/.agents/skills/README.md index e2efd80b..cea923f6 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -13,7 +13,8 @@ plus optional `references/` files. ## Scripts -Use [`../scripts/prepare-marimo-example.sh`](../scripts/prepare-marimo-example.sh) +Use [`../../scripts/convert-colab-to-marimo.py`](../../scripts/convert-colab-to-marimo.py) to create the initial marimo notebook from a Jupyter `.ipynb`, capture `marimo check` output, and write a temporary `.conversion/` report directory -for the polishing pass. +for the polishing pass. The report directory includes a Markdown handoff, +raw convert/check output, structured JSON metadata, and an event log. diff --git a/.agents/skills/marimo-wandb-notebooks/SKILL.md b/.agents/skills/marimo-wandb-notebooks/SKILL.md index e239cb89..79a2c9a0 100644 --- a/.agents/skills/marimo-wandb-notebooks/SKILL.md +++ b/.agents/skills/marimo-wandb-notebooks/SKILL.md @@ -9,15 +9,25 @@ description: Create, convert, review, or refactor repo-ready marimo example note | Situation | Do this | | --- | --- | -| Always | Read [`references/marimo-idioms.md`](references/marimo-idioms.md). | -| Starting from an existing marimo `.py` | Do not run `prepare-marimo-example.sh`. Inspect the `.py`, run `uvx marimo check `, and polish against the repo conventions below. | -| Starting from `.ipynb` | Run [`../../scripts/prepare-marimo-example.sh`](../../scripts/prepare-marimo-example.sh) ` --name `, then read [`references/conversion-cleanup.md`](references/conversion-cleanup.md) with `.conversion/conversion-report.md` and `.conversion/marimo-check.txt`. | +| Always | Read [`references/marimo-idioms.md`](references/marimo-idioms.md) and [`references/tutorial-notebook-objectives.md`](references/tutorial-notebook-objectives.md). | +| Starting from an existing marimo `.py` | Inspect the `.py` and read nearby pipeline logs (`result.json`, `marimo-check.log`, and `marimo-convert.log`) in `.logs/` first. For converted notebooks under `marimo/convert/`, also read [`references/convert-cleanup.md`](references/convert-cleanup.md) before broad cleanup. | +| Starting from `.ipynb` | Run [`../../../scripts/convert-colab-to-marimo.py`](../../../scripts/convert-colab-to-marimo.py) ` --name `, then read [`references/convert-cleanup.md`](references/convert-cleanup.md) and the generated `marimo/convert//.logs/result.json`. | | Notebook uses W&B | Read [`references/wandb-patterns.md`](references/wandb-patterns.md). | +Preserve tutorial teaching value while applying marimo conventions. + The canonical exemplar is `examples/marimo/mnist-registry/mnist_registry.py` — when in doubt, match its structure. +## Existing conversion triage + +For notebooks under `marimo/convert/`, diagnose logs before polishing. Read +`.logs/result.json`, inspect the failed stage's log, fix `marimo check` +blockers first, then run a fresh `uvx marimo check`. See +[`references/convert-cleanup.md`](references/convert-cleanup.md) for the full +triage and cleanup checklist. + ## Repo conventions - Each example lives in its own directory: `examples/marimo//`, @@ -42,45 +52,24 @@ its structure. ## Notebook structure -Order the notebook as a narrative the reader scrolls through top to bottom: - -1. **Setup cell** — `with app.setup(hide_code=True):` holds all imports, - constants, and environment detection (e.g. device selection). One place, - not scattered across cells. -2. **Intro prose** — a markdown cell (`hide_code=True`) with the title, - "What you will build", and "Prerequisites". Prose goes in markdown cells, - never mixed into code cells. -3. **`mo.outline()`** in its own cell near the top, so readers see the - notebook's shape at a glance. -4. **Configuration** — all UI controls batched into a single submittable - form (see below). -5. **The pipeline** — logic cells that consume the form, interleaved with - `## Section` markdown cells. -6. **Verify and next steps** — a closing markdown cell telling the reader - exactly what to look at (in the W&B UI: which charts, tabs, panels) and - what to try next. -7. **Helper functions** — `@app.function` / `@app.class_definition` cells - under a `## Helper functions` section at the bottom. - -## Gate execution once, then let the graph run - -Batch expensive controls into one form, gate once with `mo.stop`, and let -downstream cells depend on names defined after the gate. See -[`references/marimo-idioms.md`](references/marimo-idioms.md) for the detailed -pattern. - -## Separate logic from presentation - -Put heavy work in named helpers and keep view cells focused on rendering. See -[`references/marimo-idioms.md`](references/marimo-idioms.md) for details. +Use this marimo skeleton: setup cell, intro, outline, configuration form, +gated pipeline, verify/next steps, helper functions. See +[`references/marimo-idioms.md`](references/marimo-idioms.md) for marimo +mechanics and +[`references/tutorial-notebook-objectives.md`](references/tutorial-notebook-objectives.md) +for narrative structure. + +## Core priorities + +- Gate expensive work once, then let marimo's graph run. +- Preserve visible teaching code, especially featured W&B API calls. +- Keep notebook globals scarce; move distracting scratch work into helpers. ## Final verification - `uvx marimo check ` passes. - Globals audit: anything only used inside one step should live in a helper. -- The notebook reads top-to-bottom as a tutorial; every code cell either - shows output or is a named helper. -- Fresh-eyes test: a reader with a new W&B account can follow Prerequisites, - submit the form, and verify the result from "Verify and next steps" alone. -- `.conversion/` files are temporary debugging artifacts and must not be +- Tutorial-objectives audit passes: narrative flow, visible API examples, and + reader verification are preserved. +- `.logs/` files are temporary debugging artifacts and must not be referenced by the final notebook or docs. diff --git a/.agents/skills/marimo-wandb-notebooks/references/conversion-cleanup.md b/.agents/skills/marimo-wandb-notebooks/references/conversion-cleanup.md deleted file mode 100644 index 56b8d0a3..00000000 --- a/.agents/skills/marimo-wandb-notebooks/references/conversion-cleanup.md +++ /dev/null @@ -1,47 +0,0 @@ -# Conversion Cleanup - -Use this after `.agents/scripts/prepare-marimo-example.sh` creates the initial -marimo notebook from a Jupyter `.ipynb`. - -## Start From The Report - -- Read `.conversion/conversion-report.md` in the generated example directory. -- Read `.conversion/marimo-check.txt` before editing the notebook. -- Fix `marimo check` issues first; they often point to converted cells that - need to be split, reordered, or moved into helpers. - -## Common Converter Leftovers - -- Ensure the PEP 723 script metadata lists every runtime package the notebook - imports. The converter may miss dependencies. -- Remove Jupyter-only artifacts such as `%magic` commands, shell escapes, and - unnecessary `display()` calls. -- Make the intended output the final expression of each display cell. Indented - or conditional expressions will not render as cell output. -- Replace notebook-global scratch variables with local variables inside helper - functions when they are only used in one step. -- Prefer explicit markdown cells for prose. Do not leave tutorial text inside - code comments or string literals in logic cells. - -## Widget Cleanup - -- Replace ipywidgets with native `mo.ui` components when there is a direct - equivalent. -- Replace `interact`, `observe`, `link`, and `jslink` patterns with marimo - reactivity. Split UI definition, value consumption, and rendering into - separate cells. -- Use `mo.ui.anywidget()` only when no native marimo component fits. - -## LaTeX Cleanup - -- Use raw strings for markdown containing LaTeX, such as `mo.md(r"$x^2$")`. -- Replace MathJax-only constructs with KaTeX-compatible syntax. -- Visually verify math-heavy outputs because KaTeX failures can be quiet. - -## Final Check - -Run: - -```bash -uvx marimo check examples/marimo//.py -``` diff --git a/.agents/skills/marimo-wandb-notebooks/references/convert-cleanup.md b/.agents/skills/marimo-wandb-notebooks/references/convert-cleanup.md new file mode 100644 index 00000000..c40cb488 --- /dev/null +++ b/.agents/skills/marimo-wandb-notebooks/references/convert-cleanup.md @@ -0,0 +1,68 @@ +# Convert Cleanup + +Use this after `scripts/convert-colab-to-marimo.py` creates the initial +marimo notebook from a Jupyter `.ipynb`. The converter writes diagnostics to +`marimo/convert//.logs/`. + +## Start from convert logs + +- For batch runs, start with `marimo/convert/convert-summary.txt` to find + notebooks that need action. +- Read `marimo/convert//.logs/result.json` first. Check `status`, + `failed_stage`, `source`, `target`, and each command's exit code. +- If `failed_stage` is `convert`, read + `marimo/convert//.logs/marimo-convert.log` before editing the notebook. +- If `failed_stage` is `check`, read + `marimo/convert//.logs/marimo-check.log` before editing the notebook. +- If `status` is `ok`, still skim `result.json` to confirm the source and target + paths before cleanup. +- Fix `marimo check` issues first; they often point to converted cells that + need to be split, reordered, or moved into helpers. + +## Common Converter Leftovers + +- Follow [`tutorial-notebook-objectives.md`](tutorial-notebook-objectives.md) + when deciding which instructional cells and W&B API examples to preserve. +- Ensure the PEP 723 script metadata lists every runtime package the notebook + imports. The converter may miss dependencies. +- Remove Jupyter-only artifacts such as `%magic` commands, shell escapes, and + unnecessary `display()` calls. +- Make the intended output the final expression of each display cell. Indented + or conditional expressions will not render as cell output. +- Replace notebook-global scratch variables with local variables inside helper + functions when they are only used in one step. +- Prefer explicit markdown cells for prose. Do not leave tutorial text inside + code comments or string literals in logic cells. + +## Common Check Failures + +- Circular dependencies often come from imports or helper names returned by a + later cell and consumed by an earlier helper cell. Fix by moving shared + imports/constants into `app.setup(...)` or into the helper cell that uses + them, and avoid returning imported symbols from downstream logic cells. +- `multiple-definitions` errors can happen after moving the same import into + multiple cells. Keep cell-local imports private with an underscore alias + such as `from torch.utils.data import DataLoader as _DataLoader`. + +## Widget Cleanup + +- Replace ipywidgets with native `mo.ui` components when there is a direct + equivalent. +- Replace `interact`, `observe`, `link`, and `jslink` patterns with marimo + reactivity. Split UI definition, value consumption, and rendering into + separate cells. +- Use `mo.ui.anywidget()` only when no native marimo component fits. + +## LaTeX Cleanup + +- Use raw strings for markdown containing LaTeX, such as `mo.md(r"$x^2$")`. +- Replace MathJax-only constructs with KaTeX-compatible syntax. +- Visually verify math-heavy outputs because KaTeX failures can be quiet. + +## Final Check + +Run: + +```bash +uvx marimo check marimo/convert//.py +``` diff --git a/.agents/skills/marimo-wandb-notebooks/references/marimo-idioms.md b/.agents/skills/marimo-wandb-notebooks/references/marimo-idioms.md index 67c7b0c6..dda2bc34 100644 --- a/.agents/skills/marimo-wandb-notebooks/references/marimo-idioms.md +++ b/.agents/skills/marimo-wandb-notebooks/references/marimo-idioms.md @@ -3,13 +3,11 @@ Use this as a short review checklist when writing or polishing marimo notebooks for this repo. -## Notebook Shape +## Marimo Notebook Shape - A marimo notebook is a Python file. Cells are functions decorated with `@app.cell`; dependencies are the function arguments and return values. - Use a single setup cell for imports, constants, and environment detection. -- Add PEP 723 script metadata at the top so `uvx marimo ... --sandbox` can - recreate the runtime environment. - Keep globals scarce. Every returned name is notebook-wide, so move step-local scratch work into helper functions. @@ -26,28 +24,16 @@ for this repo. ## Gating Expensive Work -Batch every control into one form so nothing expensive runs until the user -submits: +Batch controls into one form, then gate exactly once: ```python -form = ( - mo.md( - """ - **Training.** - - {epochs} {batch_size} - ... - """ - ) - .batch(epochs=epochs, batch_size=batch_size, ...) - .form(submit_button_label="Train model", bordered=False) -) +form = mo.md("{epochs} {batch_size}").batch( + epochs=epochs, + batch_size=batch_size, +).form(submit_button_label="Train model", bordered=False) form ``` -`form.value` is `None` until submit. Gate one cell on it, with a message that -tells the reader what will happen: - ```python mo.stop( form.value is None, @@ -56,26 +42,20 @@ mo.stop( cfg = form.value ``` -Every downstream cell references names defined after the gate, such as `cfg`, -`run`, or `model`, so marimo's dependency graph holds them all back until the -form is submitted. Do not re-check the form in later cells, wrap cells in `if` -guards, or use `mo.ui.run_button()` when a form fits. +Downstream cells should depend on post-gate names such as `cfg`, `run`, or +`model`. Do not re-check the form in later cells or wrap cells in `if` guards. ## Rendering - The final expression of a cell is what renders. - Indented expressions inside `if`, `for`, `with`, or helper blocks do not become the cell output. Assign the display object, then put it last. -- Use markdown cells for prose. Use view cells for rendering. Keep heavy logic - in named helpers. +- Use markdown cells for prose. Use view cells for rendering. Keep + non-teaching heavy logic in named helpers. ## Logic And Presentation -- Heavy lifting, such as loading data, training, logging, and saving artifacts, - goes in named `@app.function` helpers. The cell body should become a short, - readable call like `model, history = run_training(...)`. -- View cells, often `hide_code=True`, render results and contain no logic worth - reading. +- View cells should keep rendering code short and avoid unrelated computation. - Push temporaries into functions to keep notebook globals to a minimum. Every returned name is reserved across the whole file. - Present results with real components, such as `mo.ui.table`, @@ -94,11 +74,3 @@ guards, or use `mo.ui.run_button()` when a form fits. - Let unexpected programming errors surface. - Catch only specific, expected failures where the notebook can give useful recovery guidance, such as W&B auth or account setup problems. - -## Verification - -Run this before handing back: - -```bash -uvx marimo check -``` diff --git a/.agents/skills/marimo-wandb-notebooks/references/wandb-patterns.md b/.agents/skills/marimo-wandb-notebooks/references/wandb-patterns.md index c7de7228..a0ead619 100644 --- a/.agents/skills/marimo-wandb-notebooks/references/wandb-patterns.md +++ b/.agents/skills/marimo-wandb-notebooks/references/wandb-patterns.md @@ -12,6 +12,21 @@ links artifacts and registry entries. ## Runs And Reruns +- When possible, initialize runs with context managers. + + ```python + import wandb + + with wandb.init() as run: + run.log({"loss": 0.1}) + ``` + + If you do not use a context manager, explicitly finish the run with + `run.finish()`. + +- Prefer run-bound methods such as `run.log`, `run.log_artifact`, and + `run.summary` unless the notebook is intentionally teaching a global API. + - marimo keeps the kernel alive across form re-submits, so finish any prior run before starting a new one: @@ -33,13 +48,13 @@ links artifacts and registry entries. ## Entity - Include an entity field. -- Explain that accounts created after May 2024 have no personal entity; the run - must go to a team. +- Some accounts require a team/entity. Explain how to find the right entity in + W&B, and make the field easy to override. ## Expected Failures - Expected failures should become guidance, not tracebacks. -- Wrap only calls that fail for account-setup reasons, such as `wandb.init` or +- Wrap only calls that fail for account-setup reasons, such as `wandb.init()` or registry linking. - Render a `mo.callout(kind="danger")` that names the likely cause and fix. - Let everything else fail naturally. Do not use `try`/`except` for normal diff --git a/.gitignore b/.gitignore index 50573998..ce90d575 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ colabs/log.txt artifacts examples/marimo/*/.conversion/ examples/jax/jax-llm/proteins-base/ +marimo/Notebook_mentions_Exported_wandb_docs_models_list.csv diff --git a/marimo/convert/convert-summary.txt b/marimo/convert/convert-summary.txt new file mode 100644 index 00000000..e19fdfd9 --- /dev/null +++ b/marimo/convert/convert-summary.txt @@ -0,0 +1,112 @@ +Convert Summary +Generated: 2026-08-18T03:52:17+00:00 +Path list: scripts/colab_sources/notebook_paths.txt + +- Total: 37 +- Needs action: 13 +- Passed: 24 + +Needs Action +- prepare_failed: colabs/keras/Keras_pipeline_with_Weights_and_Biases.ipynb + target: - + failed_stage: prepare + log: - + error: input file does not exist: colabs/keras/Keras_pipeline_with_Weights_and_Biases.ipynb + +- check_failed: colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb + target: marimo/convert/organizing-hyperparameter-sweeps-in-pytorch-with-w-b/organizing_hyperparameter_sweeps_in_pytorch_with_w_b.py + failed_stage: check + log: marimo/convert/organizing-hyperparameter-sweeps-in-pytorch-with-w-b/.logs/marimo-check.log + error: - + +- check_failed: colabs/tables/AlphaFold_with_W&B_Align,_Fold,_Log.ipynb + target: marimo/convert/alphafold-with-w-b-align-fold-log/alphafold_with_w_b_align_fold_log.py + failed_stage: check + log: marimo/convert/alphafold-with-w-b-align-fold-log/.logs/marimo-check.log + error: - + +- check_failed: colabs/torchtune/torchtune_and_wandb.ipynb + target: marimo/convert/torchtune-and-wandb/torchtune_and_wandb.py + failed_stage: check + log: marimo/convert/torchtune-and-wandb/.logs/marimo-check.log + error: - + +- check_failed: colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb + target: marimo/convert/pipeline-versioning-with-w-b-artifacts/pipeline_versioning_with_w_b_artifacts.py + failed_stage: check + log: marimo/convert/pipeline-versioning-with-w-b-artifacts/.logs/marimo-check.log + error: - + +- check_failed: colabs/wandb_registry/zoo_wandb.ipynb + target: marimo/convert/zoo-wandb/zoo_wandb.py + failed_stage: check + log: marimo/convert/zoo-wandb/.logs/marimo-check.log + error: - + +- prepare_failed: examples/boosting-algorithms/xgboost-housing/train.py + target: marimo/convert/train/train.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/boosting-algorithms/xgboost-housing/train.py + +- prepare_failed: examples/keras/keras-cnn-fashion/train.py + target: marimo/convert/train/train.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/keras/keras-cnn-fashion/train.py + +- prepare_failed: examples/pytorch/pytorch-cnn-fashion/fashion_data.py + target: marimo/convert/fashion-data/fashion_data.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/pytorch/pytorch-cnn-fashion/fashion_data.py + +- prepare_failed: examples/pytorch/pytorch-ddp/log-ddp.py + target: marimo/convert/log-ddp/log_ddp.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/pytorch/pytorch-ddp/log-ddp.py + +- prepare_failed: examples/tensorflow/tf-cnn-fashion/train.py + target: marimo/convert/train/train.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/tensorflow/tf-cnn-fashion/train.py + +- prepare_failed: examples/tensorflow/tf-estimator-mnist/mnist.py + target: marimo/convert/mnist/mnist.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/tensorflow/tf-estimator-mnist/mnist.py + +- prepare_failed: examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py + target: marimo/convert/xgboost-tune/xgboost_tune.py + failed_stage: prepare + log: - + error: input must be a .ipynb file: /Users/noahluna/Desktop/examples_group/examples/examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py + +Passed +- colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb -> marimo/convert/credit-scorecards-with-xgboost-and-w-b/credit_scorecards_with_xgboost_and_w_b.py +- colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb -> marimo/convert/using-w-b-sweeps-with-xgboost/using_w_b_sweeps_with_xgboost.py +- colabs/datasets-predictions/W&B_Tables_Quickstart.ipynb -> marimo/convert/w-b-tables-quickstart/w_b_tables_quickstart.py +- colabs/diffusers/lcm-diffusers.ipynb -> marimo/convert/lcm-diffusers/lcm_diffusers.py +- colabs/huggingface/Custom_Progress_Callback.ipynb -> marimo/convert/custom-progress-callback/custom_progress_callback.py +- colabs/huggingface/Huggingface_wandb.ipynb -> marimo/convert/huggingface-wandb/huggingface_wandb.py +- colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_&_Biases.ipynb -> marimo/convert/optimize-hugging-face-models-with-weights-biases/optimize_hugging_face_models_with_weights_biases.py +- colabs/intro/Intro_to_Weights_&_Biases.ipynb -> marimo/convert/intro-to-weights-biases/intro_to_weights_biases.py +- colabs/intro/Intro_to_Weights_&_Biases_keras.ipynb -> marimo/convert/intro-to-weights-biases-keras/intro_to_weights_biases_keras.py +- colabs/intro/Report_API_Quickstart.ipynb -> marimo/convert/report-api-quickstart/report_api_quickstart.py +- colabs/intro/run_quickstart.ipynb -> marimo/convert/run-quickstart/run_quickstart.py +- colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb -> marimo/convert/use-wandbevalcallback-in-your-keras-workflow/use_wandbevalcallback_in_your_keras_workflow.py +- colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb -> marimo/convert/use-wandbmetriclogger-in-your-keras-workflow/use_wandbmetriclogger_in_your_keras_workflow.py +- colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb -> marimo/convert/use-wandbmodelcheckpoint-in-your-keras-workflow/use_wandbmodelcheckpoint_in_your_keras_workflow.py +- colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb -> marimo/convert/openai-api-autologger-quickstart/openai_api_autologger_quickstart.py +- colabs/pytorch-lightning/Optimize_PyTorch_Lightning_models_with_Weights_&_Biases.ipynb -> marimo/convert/optimize-pytorch-lightning-models-with-weights-biases/optimize_pytorch_lightning_models_with_weights_biases.py +- colabs/pytorch/Simple_PyTorch_Integration.ipynb -> marimo/convert/simple-pytorch-integration/simple_pytorch_integration.py +- colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb -> marimo/convert/hyperparameter-optimization-in-tensorflow-using-w-b-sweeps/hyperparameter_optimization_in_tensorflow_using_w_b_sweeps.py +- colabs/wandb-artifacts/WandB_Artifacts_Time_to_live_TTL_Walkthrough.ipynb -> marimo/convert/wandb-artifacts-time-to-live-ttl-walkthrough/wandb_artifacts_time_to_live_ttl_walkthrough.py +- colabs/wandb-log/Configs_in_W&B.ipynb -> marimo/convert/configs-in-w-b/configs_in_w_b.py +- colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb -> marimo/convert/log-almost-anything-with-w-b-media/log_almost_anything_with_w_b_media.py +- colabs/wandb-log/Log_a_Confusion_Matrix_with_W&B.ipynb -> marimo/convert/log-a-confusion-matrix-with-w-b/log_a_confusion_matrix_with_w_b.py +- colabs/wandb-log/Plot_ROC_Curves_with_W&B.ipynb -> marimo/convert/plot-roc-curves-with-w-b/plot_roc_curves_with_w_b.py +- colabs/yolo/Train_and_Debug_YOLOv5_Models_with_Weights_&_Biases_.ipynb -> marimo/convert/train-and-debug-yolov5-models-with-weights-biases/train_and_debug_yolov5_models_with_weights_biases.py diff --git a/scripts/colab_sources/Notebook_mentions_Exported_wandb_docs_models_list.csv b/scripts/colab_sources/Notebook_mentions_Exported_wandb_docs_models_list.csv new file mode 100644 index 00000000..508edf73 --- /dev/null +++ b/scripts/colab_sources/Notebook_mentions_Exported_wandb_docs_models_list.csv @@ -0,0 +1,69 @@ +.mdx file,Line(s),Full path / URL,Keep? [Based on Integration-> FC],Path +docs/models/artifacts.mdx,9,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb,Yes,/wandb/examples/blob/master/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb +docs/models/artifacts/ttl.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/kas-artifacts-ttl-colab/colabs/wandb-artifacts/WandB_Artifacts_Time_to_live_TTL_Walkthrough.ipynb,Yes,/wandb/examples/blob/kas-artifacts-ttl-colab/colabs/wandb-artifacts/WandB_Artifacts_Time_to_live_TTL_Walkthrough.ipynb +docs/models/integrations/azure-openai-fine-tuning.mdx,58,,No, +docs/models/integrations/composer.mdx,7,,No, +docs/models/integrations/deepchem.mdx,31,,No, +docs/models/integrations/diffusers.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/diffusers/lcm-diffusers.ipynb,Yes,/wandb/examples/blob/master/colabs/diffusers/lcm-diffusers.ipynb +docs/models/integrations/huggingface.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Huggingface_wandb.ipynb,Yes,/wandb/examples/blob/master/colabs/huggingface/Huggingface_wandb.ipynb +docs/models/integrations/huggingface_transformers.mdx,9,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_&_Biases.ipynb,Yes,/wandb/examples/blob/master/colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_&_Biases.ipynb +docs/models/integrations/huggingface_transformers.mdx,436,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/huggingface/Custom_Progress_Callback.ipynb,Yes,/wandb/examples/blob/master/colabs/huggingface/Custom_Progress_Callback.ipynb +docs/models/integrations/keras.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/intro/Intro_to_Weights_%26_Biases_keras.ipynb,Yes,/wandb/examples/blob/master/colabs/intro/Intro_to_Weights_%26_Biases_keras.ipynb +docs/models/integrations/keras.mdx,36,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb,Yes,/wandb/examples/blob/master/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb +docs/models/integrations/keras.mdx,74,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb,Yes,/wandb/examples/blob/master/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb +docs/models/integrations/keras.mdx,149,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb,Yes,/wandb/examples/blob/master/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb +docs/models/integrations/keras.mdx,260,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/Keras_pipeline_with_Weights_and_Biases.ipynb,Yes,/wandb/examples/blob/master/colabs/keras/Keras_pipeline_with_Weights_and_Biases.ipynb +docs/models/integrations/keras.mdx,262,https://github.com/wandb/examples/blob/master/examples/keras/keras-cnn-fashion/train.py,Yes,/wandb/examples/blob/master/examples/keras/keras-cnn-fashion/train.py +docs/models/integrations/lightgbm.mdx,,,No, +docs/models/integrations/lightgbm.mdx,,,No, +docs/models/integrations/lightgbm.mdx,,,No, +docs/models/integrations/lightning.mdx,11,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/pytorch-lightning/Optimize_PyTorch_Lightning_models_with_Weights_%26_Biases.ipynb,Yes,/wandb/examples/blob/master/colabs/pytorch-lightning/Optimize_PyTorch_Lightning_models_with_Weights_%26_Biases.ipynb +docs/models/integrations/openai-api.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb,Yes,/wandb/examples/blob/master/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb +docs/models/integrations/pytorch.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/pytorch/Simple_PyTorch_Integration.ipynb,Yes,/wandb/examples/blob/master/colabs/pytorch/Simple_PyTorch_Integration.ipynb +docs/models/integrations/sagemaker.mdx,,,No, +docs/models/integrations/tensorboard.mdx,,,No, +docs/models/integrations/tensorflow.mdx,101,https://github.com/wandb/examples/blob/master/examples/tensorflow/tf-estimator-mnist/mnist.py,Check,/wandb/examples/blob/master/examples/tensorflow/tf-estimator-mnist/mnist.py +docs/models/integrations/tensorflow.mdx,102,https://github.com/wandb/examples/blob/master/examples/tensorflow/tf-cnn-fashion/train.py,Check,/wandb/examples/blob/master/examples/tensorflow/tf-cnn-fashion/train.py +docs/models/integrations/torchtune.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/torchtune/torchtune_and_wandb.ipynb,Check,/wandb/examples/blob/master/colabs/torchtune/torchtune_and_wandb.ipynb +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/ultralytics.mdx,,,No, +docs/models/integrations/xgboost.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W%26B.ipynb,Yes,/wandb/examples/blob/master/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W%26B.ipynb +docs/models/integrations/xgboost.mdx,62,https://github.com/wandb/examples/tree/master/examples/boosting-algorithms,Yes,/wandb/examples/blob/main/examples/boosting-algorithms/xgboost-housing/train.py +docs/models/integrations/xgboost.mdx,70,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/boosting/Using_W%26B_Sweeps_with_XGBoost.ipynb,Yes,/wandb/examples/blob/master/colabs/boosting/Using_W%26B_Sweeps_with_XGBoost.ipynb +docs/models/integrations/xgboost.mdx,72,https://github.com/wandb/examples/blob/master/examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py,Yes,/wandb/examples/blob/master/examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py +docs/models/integrations/yolov5.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/yolo/Train_and_Debug_YOLOv5_Models_with_Weights_%26_Biases_.ipynb,Yes,/wandb/examples/blob/master/colabs/yolo/Train_and_Debug_YOLOv5_Models_with_Weights_%26_Biases_.ipynb +docs/models/quickstart.mdx,11,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/intro/run_quickstart.ipynb,,/wandb/examples/blob/master/colabs/intro/run_quickstart.ipynb +docs/models/quickstart.mdx,12,,, +docs/models/ref/wandb_workspaces.mdx,58,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/intro/Report_API_Quickstart.ipynb,,/wandb/examples/blob/master/colabs/intro/Report_API_Quickstart.ipynb +docs/models/registry.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb_registry/zoo_wandb.ipynb,,/wandb/examples/blob/master/colabs/wandb_registry/zoo_wandb.ipynb +docs/models/registry/aliases.mdx,133,,, +docs/models/registry/aliases.mdx,133,,, +docs/models/reports.mdx,10,,, +docs/models/reports/create-a-report.mdx,13,,, +docs/models/sweeps.mdx,9,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W%26B.ipynb,,/wandb/examples/blob/master/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W%26B.ipynb +docs/models/sweeps.mdx,57,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb,,/wandb/examples/blob/master/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb +docs/models/sweeps.mdx,58,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb,,/wandb/examples/blob/master/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb +docs/models/sweeps/existing-project.mdx,8,https://github.com/wandb/examples/tree/master/examples/pytorch/pytorch-cnn-fashion,,/wandb/examples/blob/main/examples/pytorch/pytorch-cnn-fashion/fashion_data.py +docs/models/sweeps/useful-resources.mdx,35,https://github.com/wandb/examples/blob/master/examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py,,/wandb/examples/blob/master/examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py +docs/models/tables.mdx,9,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/datasets-predictions/W%26B_Tables_Quickstart.ipynb,,/wandb/examples/blob/master/colabs/datasets-predictions/W%26B_Tables_Quickstart.ipynb +docs/models/tables.mdx,10,https://wandb.ai/wandb/examples/reports/AlphaFold-ed-Proteins-in-W-B-Tables--Vmlldzo4ODc0MDc,,/wandb/examples/blob/master/colabs/tables/AlphaFold_with_W%26B_Align%2C_Fold%2C_Log.ipynb +docs/models/tables/tables-walkthrough.mdx,12,,, +docs/models/tables/visualize-tables.mdx,92,,, +docs/models/tables/visualize-tables.mdx,206,,, +docs/models/track.mdx,9,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/intro/Intro_to_Weights_%26_Biases.ipynb,,/wandb/examples/blob/master/colabs/intro/Intro_to_Weights_%26_Biases.ipynb +docs/models/track/config.mdx,7,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Configs_in_W%26B.ipynb,,/wandb/examples/blob/master/colabs/wandb-log/Configs_in_W%26B.ipynb +docs/models/track/log/distributed-training.mdx,26,,, +docs/models/track/log/distributed-training.mdx,"26, 35",https://github.com/wandb/examples/blob/master/examples/pytorch/pytorch-ddp/log-ddp.py,,/wandb/examples/blob/master/examples/pytorch/pytorch-ddp/log-ddp.py +docs/models/track/log/distributed-training.mdx,40,,, +docs/models/track/log/log-models.mdx,7,,, +docs/models/track/log/media.mdx,8,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Log_(Almost)_Anything_with_W%26B_Media.ipynb,,/wandb/examples/blob/master/colabs/wandb-log/Log_(Almost)_Anything_with_W%26B_Media.ipynb +docs/models/track/log/plots.mdx,210,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Plot_ROC_Curves_with_W%26B.ipynb,,/wandb/examples/blob/master/colabs/wandb-log/Plot_ROC_Curves_with_W%26B.ipynb +docs/models/track/log/plots.mdx,238,https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Log_a_Confusion_Matrix_with_W%26B.ipynb,,/wandb/examples/blob/master/colabs/wandb-log/Log_a_Confusion_Matrix_with_W%26B.ipynb +,,,, +,,,, +,,,, +,,42,, \ No newline at end of file diff --git a/scripts/colab_sources/make_list.py b/scripts/colab_sources/make_list.py new file mode 100644 index 00000000..689006a7 --- /dev/null +++ b/scripts/colab_sources/make_list.py @@ -0,0 +1,43 @@ +"""Build and normalize marimo notebook path lists.""" + +import argparse +import csv +from urllib.parse import unquote + + +def normalize_path(path): + """Convert a GitHub blob path from the source list into a repo-local path.""" + path = unquote(path.strip()) + prefix = "/wandb/examples/blob/" + if not path.startswith(prefix): + return path + return path[len(prefix):].split("/", 1)[1] + + +def create_path_list_file(args): + """Read the CSV Path column and write unique normalized paths, one per line.""" + with open(args.input_file, newline="", encoding="utf-8") as f: + paths = { + normalize_path(row["Path"]) + for row in csv.DictReader(f) + if row.get("Path") + } + + with open(args.output_file, "w") as f: + f.write("\n".join(sorted(paths))) + f.write("\n") + + +def main(args): + if not args.input_file: + raise SystemExit("--input_file is required") + + create_path_list_file(args) + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser(description="Extract path column from CSV and return a list of unique paths.") + argparser.add_argument("--input_file", help="Path to the input CSV file.") + argparser.add_argument("--output_file", default="notebook_paths.txt", help="Path to the output file.") + args = argparser.parse_args() + main(args) diff --git a/scripts/colab_sources/notebook_paths.txt b/scripts/colab_sources/notebook_paths.txt new file mode 100644 index 00000000..5e2db8e6 --- /dev/null +++ b/scripts/colab_sources/notebook_paths.txt @@ -0,0 +1,37 @@ +colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb +colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb +colabs/datasets-predictions/W&B_Tables_Quickstart.ipynb +colabs/diffusers/lcm-diffusers.ipynb +colabs/huggingface/Custom_Progress_Callback.ipynb +colabs/huggingface/Huggingface_wandb.ipynb +colabs/huggingface/Optimize_Hugging_Face_models_with_Weights_&_Biases.ipynb +colabs/intro/Intro_to_Weights_&_Biases.ipynb +colabs/intro/Intro_to_Weights_&_Biases_keras.ipynb +colabs/intro/Report_API_Quickstart.ipynb +colabs/intro/run_quickstart.ipynb +colabs/keras/Keras_pipeline_with_Weights_and_Biases.ipynb +colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb +colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb +colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb +colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb +colabs/pytorch-lightning/Optimize_PyTorch_Lightning_models_with_Weights_&_Biases.ipynb +colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb +colabs/pytorch/Simple_PyTorch_Integration.ipynb +colabs/tables/AlphaFold_with_W&B_Align,_Fold,_Log.ipynb +colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb +colabs/torchtune/torchtune_and_wandb.ipynb +colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb +colabs/wandb-artifacts/WandB_Artifacts_Time_to_live_TTL_Walkthrough.ipynb +colabs/wandb-log/Configs_in_W&B.ipynb +colabs/wandb-log/Log_(Almost)_Anything_with_W&B_Media.ipynb +colabs/wandb-log/Log_a_Confusion_Matrix_with_W&B.ipynb +colabs/wandb-log/Plot_ROC_Curves_with_W&B.ipynb +colabs/wandb_registry/zoo_wandb.ipynb +colabs/yolo/Train_and_Debug_YOLOv5_Models_with_Weights_&_Biases_.ipynb +examples/boosting-algorithms/xgboost-housing/train.py +examples/keras/keras-cnn-fashion/train.py +examples/pytorch/pytorch-cnn-fashion/fashion_data.py +examples/pytorch/pytorch-ddp/log-ddp.py +examples/tensorflow/tf-cnn-fashion/train.py +examples/tensorflow/tf-estimator-mnist/mnist.py +examples/wandb-sweeps/sweeps-xgboost/xgboost_tune.py diff --git a/scripts/convert-colab-to-marimo.py b/scripts/convert-colab-to-marimo.py new file mode 100755 index 00000000..ce49612e --- /dev/null +++ b/scripts/convert-colab-to-marimo.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""Convert Jupyter notebooks into marimo examples. + +Convert one notebook at a time, or in batch from a list of paths. Creates +per-notebook diagnostics in marimo/convert//.logs/. Batch runs also +write marimo/convert/convert-summary.txt. + +Usage: + convert-colab-to-marimo.py notebook.ipynb --name example-name + convert-colab-to-marimo.py notebook.ipynb --name example-name --force + convert-colab-to-marimo.py notebook-path-list.txt --batch +""" + +from __future__ import annotations + +import argparse +import json +import re +import shlex +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Sequence + +MARIMO_DIR = Path("marimo") +CONVERT_DIR = MARIMO_DIR / "convert" +BATCH_SUMMARY = CONVERT_DIR / "convert-summary.txt" + + +class PrepareError(RuntimeError): + """User-facing setup or input error.""" + + +@dataclass +class BatchResult: + """One notebook entry in the batch summary.""" + + raw_path: str + status: str + source: Path | None = None + target: Path | None = None + failed_stage: str | None = None + log: Path | None = None + error: str | None = None + + +def validate_repo_root(repo_root: Path) -> None: + """Validate that ``repo_root`` contains the marimo examples directory. + + Args: + repo_root: Expected root of the examples repository. + + Raises: + PrepareError: If ``repo_root`` does not contain ``marimo/``. + """ + + if not (repo_root / MARIMO_DIR).is_dir(): + raise PrepareError( + "Run this script from the root of the wandb/examples repository " + "(the directory containing examples/marimo)." + ) + + +def resolve_file(raw_path: str, *, base_dir: Path) -> Path: + """Resolve an input file path. + + Args: + raw_path: Absolute path, user path, or path relative to ``base_dir``. + base_dir: Directory used to resolve relative paths. + + Returns: + Absolute resolved path to an existing file. + + Raises: + PrepareError: If the resolved path is not a file. + """ + + path = Path(raw_path).expanduser() + if not path.is_absolute(): + path = base_dir / path + path = path.resolve() + + if not path.is_file(): + raise PrepareError(f"input file does not exist: {raw_path}") + return path + + +def display_path(path: Path, repo_root: Path) -> str: + """Format a path for logs and terminal output. + + Args: + path: Path to display. + repo_root: Repository root used for relative display. + + Returns: + Repository-relative path when possible; otherwise an absolute path. + """ + + try: + return str(path.resolve().relative_to(repo_root.resolve())) + except ValueError: + return str(path.resolve()) + + +def slug_from_notebook(notebook: Path) -> str: + """Create a batch-mode example name from a notebook filename. + + Args: + notebook: Source notebook path. + + Returns: + Lowercase, dash-separated slug derived from the notebook stem. + + Raises: + PrepareError: If the filename cannot produce a non-empty slug. + """ + + slug = re.sub(r"[^a-z0-9]+", "-", notebook.stem.lower()).strip("-") + if not slug: + raise PrepareError(f"could not derive a name from notebook: {notebook}") + return slug + + +def validate_name(name: str) -> None: + """Validate a caller-provided marimo example name. + + Args: + name: Target directory name under ``marimo/convert``. + + Raises: + PrepareError: If ``name`` contains unsupported characters. + """ + + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", name): + raise PrepareError( + "--name must start with a letter or number and contain only " + f"letters, numbers, '-' or '_': {name}" + ) + + +def target_paths(name: str, *, repo_root: Path) -> tuple[Path, Path, Path]: + """Return target directory, notebook file, and log directory for ``name``. + + Args: + name: Target example name under ``marimo/convert``. + repo_root: Repository root used to resolve output paths. + + Returns: + Tuple of ``(target_dir, target_file, debug_dir)``. + """ + + target_dir = repo_root / CONVERT_DIR / name + target = target_dir / f"{name.replace('-', '_')}.py" + debug_dir = target_dir / ".logs" + return target_dir, target, debug_dir + + +def run_command(argv: Sequence[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + """Run a command and capture stdout and stderr together. + + Args: + argv: Command and arguments to execute. + cwd: Working directory for the command. + + Returns: + Completed process, including a 127 return code if the executable is + missing. + """ + + try: + return subprocess.run( + list(argv), + cwd=cwd, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + except FileNotFoundError as error: + return subprocess.CompletedProcess( + args=list(argv), + returncode=127, + stdout=f"command not found: {argv[0]}\n{error}\n", + ) + + +def write_command_log(path: Path, result: subprocess.CompletedProcess[str]) -> None: + """Write a command transcript for a failed run. + + Args: + path: Output log file path. + result: Completed process to serialize. + """ + + output = result.stdout or "" + text = ( + f"$ {shlex.join(result.args)}\n\n" + f"exit_code: {result.returncode}\n\n" + "--- output ---\n\n" + f"{output}" + ) + if output and not output.endswith("\n"): + text += "\n" + path.write_text(text, encoding="utf-8") + + +def write_diagnostics( + *, + debug_dir: Path, + source: Path, + target: Path, + repo_root: Path, + status: str, + failed_stage: str | None, + commands: dict[str, subprocess.CompletedProcess[str]], +) -> None: + """Write convert diagnostics to a JSON file and failure logs. + + Args: + debug_dir: Directory for ``result.json`` and failure logs. + source: Source notebook path. + target: Generated marimo Python file path. + repo_root: Repository root used for display paths. + status: Final status string for the convert attempt. + failed_stage: Stage name that failed, if any. + commands: Completed commands keyed by stage name. + """ + + # Successful reruns should not leave old failure logs behind. + for old_log in debug_dir.glob("marimo-*.log"): + old_log.unlink() + + command_metadata: dict[str, dict[str, object]] = {} + for stage, result in commands.items(): + metadata: dict[str, object] = { + "command": list(result.args), + "exit_code": result.returncode, + } + + if failed_stage is not None: + log_path = debug_dir / f"marimo-{stage}.log" + write_command_log(log_path, result) + metadata["log"] = display_path(log_path, repo_root) + + command_metadata[stage] = metadata + + result = { + "completed_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "status": status, + "failed_stage": failed_stage, + "source": display_path(source, repo_root), + "target": display_path(target, repo_root), + "target_exists": target.exists(), + "commands": command_metadata, + } + (debug_dir / "result.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def prepare_notebook( + source: Path, + name: str, + *, + repo_root: Path, + force: bool, +) -> str: + """Convert and check one notebook. + + Uses ``uvx marimo convert`` and ``uvx marimo check``. + + Args: + source: Source ``.ipynb`` file. + name: Target example name under ``marimo/convert``. + repo_root: Repository root for output paths and command execution. + force: Whether to overwrite an existing target. + + Returns: + One of ``"ok"``, ``"convert_failed"``, or ``"check_failed"``. + + Raises: + PrepareError: If the input or target name is invalid. + """ + + if source.suffix.lower() != ".ipynb": + raise PrepareError(f"input must be a .ipynb file: {source}") + validate_name(name) + + target_dir, target, debug_dir = target_paths(name, repo_root=repo_root) + + if target.exists() and not force: + raise PrepareError(f"target already exists: {display_path(target, repo_root)}") + + target_dir.mkdir(parents=True, exist_ok=True) + debug_dir.mkdir(parents=True, exist_ok=True) + + commands: dict[str, subprocess.CompletedProcess[str]] = {} + + print(f"Converting {display_path(source, repo_root)}...", flush=True) + commands["convert"] = run_command( + ["uvx", "marimo", "convert", str(source), "-o", str(target)], + cwd=repo_root, + ) + + if commands["convert"].returncode != 0: + status = "convert_failed" + failed_stage = "convert" + else: + print(f"Checking {display_path(target, repo_root)}...", flush=True) + commands["check"] = run_command( + ["uvx", "marimo", "check", str(target)], + cwd=repo_root, + ) + failed_stage = "check" if commands["check"].returncode != 0 else None + status = "check_failed" if failed_stage else "ok" + + write_diagnostics( + debug_dir=debug_dir, + source=source, + target=target, + repo_root=repo_root, + status=status, + failed_stage=failed_stage, + commands=commands, + ) + + if failed_stage: + log_path = debug_dir / f"marimo-{failed_stage}.log" + print(f"{status.replace('_', ' ')}. See {display_path(log_path, repo_root)}", file=sys.stderr) + else: + print(f"Prepared {display_path(target, repo_root)}") + + return status + + +def iter_path_list(path_list: Path) -> list[str]: + """Read notebook paths from a batch file. + + Args: + path_list: Text file with one notebook path per line. + + Returns: + Non-empty, non-comment path entries. + """ + + return [ + line.strip() + for line in path_list.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + +def write_batch_summary( + *, + summary_path: Path, + path_list: Path, + repo_root: Path, + results: Sequence[BatchResult], +) -> None: + """Write a human-readable summary for a batch convert. + + Args: + summary_path: Text file to write. + path_list: Batch input file used for the run. + repo_root: Repository root used for display paths. + results: Per-notebook batch results. + """ + + needs_action = [result for result in results if result.status != "ok"] + passed = [result for result in results if result.status == "ok"] + + lines = [ + "Convert Summary", + f"Generated: {datetime.now(timezone.utc).isoformat(timespec='seconds')}", + f"Path list: {display_path(path_list, repo_root)}", + "", + f"- Total: {len(results)}", + f"- Needs action: {len(needs_action)}", + f"- Passed: {len(passed)}", + "", + "Needs Action", + ] + + if needs_action: + for result in needs_action: + source = result.source if result.source is not None else result.raw_path + target = display_path(result.target, repo_root) if result.target else "" + log = display_path(result.log, repo_root) if result.log else "" + source_text = ( + display_path(source, repo_root) if isinstance(source, Path) else source + ) + lines.extend( + [ + f"- {result.status}: {source_text}", + f" target: {target or '-'}", + f" failed_stage: {result.failed_stage or '-'}", + f" log: {log or '-'}", + f" error: {result.error or '-'}", + "", + ] + ) + else: + lines.append("No notebooks need action.") + lines.append("") + + lines.append("Passed") + + if passed: + for result in passed: + source = ( + display_path(result.source, repo_root) + if result.source + else result.raw_path + ) + target = display_path(result.target, repo_root) if result.target else "" + lines.append(f"- {source} -> {target}") + else: + lines.append("No notebooks passed.") + + summary_path.parent.mkdir(parents=True, exist_ok=True) + summary_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run_batch( + path_list: Path, + *, + repo_root: Path, + force: bool, + fail_on_check: bool, +) -> int: + """Convert every notebook listed in a batch file. + + Args: + path_list: Text file containing notebook paths. + repo_root: Repository root used for resolving batch entries. + force: Whether to overwrite existing targets. + fail_on_check: Whether check failures should make the batch fail. + + Returns: + ``0`` when all required stages pass; otherwise ``1``. + """ + + had_failure = False + results: list[BatchResult] = [] + + for raw_path in iter_path_list(path_list): + source: Path | None = None + target: Path | None = None + name: str | None = None + + try: + # Batch entries are always relative to the repository root. + source = resolve_file(raw_path, base_dir=repo_root) + name = slug_from_notebook(source) + _, target, debug_dir = target_paths(name, repo_root=repo_root) + status = prepare_notebook( + source, + name, + repo_root=repo_root, + force=force, + ) + except PrepareError as error: + print(error, file=sys.stderr) + had_failure = True + if name is not None and target is None: + _, target, _ = target_paths(name, repo_root=repo_root) + results.append( + BatchResult( + raw_path=raw_path, + source=source, + target=target, + status="prepare_failed", + failed_stage="prepare", + error=str(error), + ) + ) + continue + + failed_stage = None + log = None + if status == "convert_failed": + failed_stage = "convert" + log = debug_dir / "marimo-convert.log" + elif status == "check_failed": + failed_stage = "check" + log = debug_dir / "marimo-check.log" + + results.append( + BatchResult( + raw_path=raw_path, + source=source, + target=target, + status=status, + failed_stage=failed_stage, + log=log, + ) + ) + + if status == "convert_failed" or (status == "check_failed" and fail_on_check): + had_failure = True + + summary_path = repo_root / BATCH_SUMMARY + write_batch_summary( + summary_path=summary_path, + path_list=path_list, + repo_root=repo_root, + results=results, + ) + print(f"Batch summary: {display_path(summary_path, repo_root)}") + + return int(had_failure) + + +def build_parser() -> argparse.ArgumentParser: + """Build the command-line parser. + + Returns: + Parser for single-notebook and batch convert modes. + """ + parser = argparse.ArgumentParser( + description="Convert Jupyter notebooks to marimo examples with minimal diagnostics." + ) + parser.add_argument("input", help="A .ipynb notebook, or a path-list file with --batch.") + parser.add_argument("--name", help="Target example name for a single notebook.") + parser.add_argument("--batch", action="store_true", help="Treat input as a notebook path list.") + parser.add_argument("--force", action="store_true", help="Overwrite an existing target.") + parser.add_argument( + "--fail-on-check", + action="store_true", + help="Exit non-zero when marimo check reports issues.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Convert one notebook or a batch of notebooks.""" + + args = build_parser().parse_args(argv) + repo_root = Path.cwd().resolve() + + try: + validate_repo_root(repo_root) + + if args.batch: + if args.name: + raise PrepareError("--name cannot be used with --batch") + path_list = resolve_file(args.input, base_dir=repo_root) + return run_batch( + path_list, + repo_root=repo_root, + force=args.force, + fail_on_check=args.fail_on_check, + ) + + if not args.name: + raise PrepareError("--name is required unless --batch is used") + + source = resolve_file(args.input, base_dir=repo_root) + status = prepare_notebook( + source, + args.name, + repo_root=repo_root, + force=args.force, + ) + + if status == "convert_failed": + return 1 + if status == "check_failed" and args.fail_on_check: + return 1 + return 0 + + except PrepareError as error: + print(error, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())