From 5c517675e56599d84f1a845da1869512d339d3cc Mon Sep 17 00:00:00 2001 From: ChrisNick92 Date: Mon, 20 Apr 2026 14:03:12 +0300 Subject: [PATCH] (issue #63) add verbosity on training and evaluation procedures through the flag --- pyproject.toml | 2 +- src/deepaudiox/__init__.py | 2 +- src/deepaudiox/callbacks/checkpointer.py | 14 ++-- src/deepaudiox/callbacks/console_logger.py | 79 ---------------------- src/deepaudiox/inference/__init__.py | 0 src/deepaudiox/loops/evaluator.py | 25 ++++--- src/deepaudiox/loops/trainer.py | 58 +++++++++++++--- uv.lock | 2 +- 8 files changed, 77 insertions(+), 105 deletions(-) delete mode 100644 src/deepaudiox/callbacks/console_logger.py delete mode 100644 src/deepaudiox/inference/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 3c8aba9..83b19b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "deepaudio-x" -version = "0.4.3" +version = "0.4.4" description = "DeepAudio-X: Self-supervised audio toolkit for audio classification and beyond." authors = [ { name = "Christos Nikou", email = "chrisnick92@gmail.com" }, diff --git a/src/deepaudiox/__init__.py b/src/deepaudiox/__init__.py index 8886f53..2357984 100644 --- a/src/deepaudiox/__init__.py +++ b/src/deepaudiox/__init__.py @@ -2,7 +2,7 @@ This page provides the core API reference for DeepAudioX. """ -__version__ = "0.4.3" +__version__ = "0.4.4" from deepaudiox.datasets.audio_classification_dataset import ( # noqa: F401 AudioClassificationDataset, diff --git a/src/deepaudiox/callbacks/checkpointer.py b/src/deepaudiox/callbacks/checkpointer.py index f7bb5ba..0c114df 100644 --- a/src/deepaudiox/callbacks/checkpointer.py +++ b/src/deepaudiox/callbacks/checkpointer.py @@ -44,11 +44,12 @@ def on_epoch_end(self, trainer): if trainer.state.lowest_loss > latest_validation_loss: decrease_percentage = (trainer.state.lowest_loss - latest_validation_loss) / trainer.state.lowest_loss * 100 - self.logger.info( - f"[CHECKPOINTER] Validation loss decreased: " - f"({trainer.state.lowest_loss:.6f} --> {latest_validation_loss:.6f}), " - f"{GREEN}(-{decrease_percentage:.2f}%){ENDC}." - ) + if trainer.verbose: + self.logger.info( + f"[CHECKPOINTER] Validation loss decreased: " + f"({trainer.state.lowest_loss:.6f} --> {latest_validation_loss:.6f}), " + f"{GREEN}(-{decrease_percentage:.2f}%){ENDC}." + ) trainer.state.lowest_loss = latest_validation_loss @@ -61,7 +62,8 @@ def on_epoch_end(self, trainer): }, self.path_to_checkpoint, ) - self.logger.info(f"[CHECKPOINTER] Checkpoint saved successfully at: {self.path_to_checkpoint}") + if trainer.verbose: + self.logger.info(f"[CHECKPOINTER] Checkpoint saved successfully at: {self.path_to_checkpoint}") except PermissionError: self.logger.info(f"[CHECKPOINTER] Permission denied: cannot write to {self.path_to_checkpoint}") except FileNotFoundError: diff --git a/src/deepaudiox/callbacks/console_logger.py b/src/deepaudiox/callbacks/console_logger.py deleted file mode 100644 index 38fab28..0000000 --- a/src/deepaudiox/callbacks/console_logger.py +++ /dev/null @@ -1,79 +0,0 @@ -import logging -import time - -from deepaudiox.callbacks.base_callback import BaseCallback -from deepaudiox.utils.training_utils import get_logger - - -class ConsoleLogger(BaseCallback): - """Training callback for logging messages in the console. - - Log messages in the console throughout the training process. - - Attributes: - logger: A module for logging messages. - - """ - - def __init__(self, logger: logging.Logger | None = None): - """Initialize the callback. - - Args: - logger (logging.Logger): A module for logging messages. Defaults to None. - last_time (float): Keeps track of epoch duration. - - """ - self.last_time = time.time() - self.logger = logger or get_logger() - - def on_epoch_start(self, trainer): - """When epoch starts, log indicative message. - - Args: - trainer (trainer.Trainer): The training module of the SDK. - """ - self.logger.info(f"[Epoch {trainer.state.current_epoch}/{trainer.epochs}]") - - def on_epoch_end(self, trainer): - """When epoch ends, log epoch duration and recorded scores. - - Args: - trainer (trainer.Trainer): The training module of the SDK. - - """ - elapsed_time = time.time() - self.last_time - train_loss = trainer.state.train_loss[-1] - validation_loss = trainer.state.validation_loss[-1] - - self.logger.info( - f"Epoch {trainer.state.current_epoch} | " - f"Train Loss: {train_loss:.4f} | " - f"Val. Loss: {validation_loss:.4f} | " - f"Time: {elapsed_time:.2f}s" - ) - - self.last_time = time.time() - - return - - def on_train_end(self, trainer): - """When train ends, show indicative message. - - Args: - trainer (trainer.Trainer): The training module of the SDK. - - """ - self.logger.info("Training has finished.") - - return - - def on_testing_end(self, evaluator): - """When testing ends, show indicative message. - - Args: - evaluator (evaluator.Evaluator): The evaluation module of the SDK. - - """ - self.logger.info("Testing has finished.") - - return diff --git a/src/deepaudiox/inference/__init__.py b/src/deepaudiox/inference/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/deepaudiox/loops/evaluator.py b/src/deepaudiox/loops/evaluator.py index 9d20af4..555cc7d 100644 --- a/src/deepaudiox/loops/evaluator.py +++ b/src/deepaudiox/loops/evaluator.py @@ -5,7 +5,6 @@ from torch.utils.data import DataLoader from tqdm import tqdm -from deepaudiox.callbacks.console_logger import ConsoleLogger from deepaudiox.callbacks.reporter import Reporter from deepaudiox.datasets.audio_classification_dataset import AudioClassificationDataset from deepaudiox.modules.baseclasses import BaseAudioClassifier @@ -36,11 +35,12 @@ class Evaluator: Attributes: state (State): Stores testing variables. + verbose (bool): Whether to log the evaluation report after testing. device (str): The device used for testing. class_mapping (dict): A mapping between class names and IDs. logger (logging.Logger): A module used for logging messages. test_dloader (torch.DataLoader): The DataLoader of the testing set. - model (BaseAudioClassifier): An AudioClassifier module inhereting from BaseAudioClassifier. + model (BaseAudioClassifier): An AudioClassifier module inheriting from BaseAudioClassifier. callbacks (list): A list of callbacks used throughout the testing lifecycle. """ @@ -53,12 +53,13 @@ def __init__( num_workers: int = 4, device: DeviceName = "cuda", device_index: int | None = None, + verbose: bool = True, ): """Initialize the Evaluator. Args: test_dset (AudioClassificationDataset): The testing dataset. - model (BaseAudioClassifier): An AudioClassifier module inhereting from BaseAudioClassifier. + model (BaseAudioClassifier): An AudioClassifier module inheriting from BaseAudioClassifier. class_mapping (dict): A mapping between class names and IDs. batch_size (int, optional): The batch size for Python Data Loaders. Defaults to 16. num_workers (int, optional): The number of workers for Python Data Loaders. Defaults to 4. @@ -66,6 +67,8 @@ def __init__( Defaults to ``"cuda"``. device_index (int | None): The GPU device index. Only applicable when ``device="cuda"``. If ``None``, uses the default CUDA device. + verbose (bool): If True, prints the classification report, confusion matrix, and average + posteriors after evaluation. "Evaluation has finished." is always printed. Defaults to True. Example: >>> import torch @@ -83,6 +86,7 @@ def __init__( >>> evaluator.evaluate() """ self.state = State() + self.verbose = verbose self.device = get_device(device=device, device_index=device_index) self.class_mapping = class_mapping @@ -106,7 +110,7 @@ def __init__( self.model.eval() # Configure callbacks - self.callbacks = [ConsoleLogger(logger=self.logger), Reporter(logger=self.logger)] + self.callbacks = [Reporter(logger=self.logger)] @torch.inference_mode() def evaluate(self) -> None: @@ -116,6 +120,10 @@ def evaluate(self) -> None: predicted labels, and posterior probabilities into ``self.state``, then triggers the registered callbacks via ``on_testing_end``. + Always prints "Evaluation has finished." regardless of ``verbose``. + The ``Reporter`` callback (classification report, confusion matrix, average + posteriors) is only executed when ``verbose=True``. + After this method returns, ``self.state`` holds: - ``y_true`` (np.ndarray): Ground-truth class indices, shape (N,). - ``y_pred`` (np.ndarray): Predicted class indices, shape (N,). @@ -124,7 +132,6 @@ def evaluate(self) -> None: Note: The model is expected to already be in eval mode (set in ``__init__``). Runs under ``torch.inference_mode()`` — gradients are fully disabled. - Callbacks (``ConsoleLogger``, ``Reporter``) are executed once after the loop. """ # Lists to accumulate evaluation results, i.e., true_labels, prediction_labels, and posteriors y_true_batches, y_pred_batches, posterior_batches = [], [], [] @@ -149,6 +156,8 @@ def evaluate(self) -> None: self.state.y_pred = np.concatenate(y_pred_batches) self.state.posteriors = np.concatenate(posterior_batches) - # Execute callbacks at the end of testing - for cb in self.callbacks: - cb.on_testing_end(self) + self.logger.info("Evaluation has finished.") + + if self.verbose: + for cb in self.callbacks: + cb.on_testing_end(self) diff --git a/src/deepaudiox/loops/trainer.py b/src/deepaudiox/loops/trainer.py index f413591..f550171 100644 --- a/src/deepaudiox/loops/trainer.py +++ b/src/deepaudiox/loops/trainer.py @@ -1,3 +1,4 @@ +import time from dataclasses import dataclass, field import numpy as np @@ -9,7 +10,6 @@ from tqdm import tqdm from deepaudiox.callbacks.checkpointer import Checkpointer -from deepaudiox.callbacks.console_logger import ConsoleLogger from deepaudiox.callbacks.early_stopper import EarlyStopper from deepaudiox.datasets.audio_classification_dataset import AudioClassificationDataset from deepaudiox.modules.baseclasses import BaseAudioClassifier @@ -46,15 +46,16 @@ class Trainer: Attributes: state (State): Stores training variables. epochs (int): The maximum number of training epochs. + verbose (bool): Whether to log epoch-level artifacts. device (str): The device used for training. logger (logging.Logger): A module used for logging messages. train_dloader (torch.DataLoader): The DataLoader of the training set. validation_dloader (torch.DataLoader): The DataLoader of the validation set. model (BaseAudioClassifier): The BaseAudioClassifier to be trained. optimizer (torch.optim.Optimizer): The optimizer of the training process. - lr_scheduler (torch.optim.Optimizer): The scheduler of the training process. + scheduler (LRScheduler): The learning rate scheduler of the training process. loss_function (nn.Module): The loss function used for optimization. - callbacks (list): A list of callbacks used throught the training lifecycle. + callbacks (list): A list of callbacks used throughout the training lifecycle. """ @@ -75,6 +76,7 @@ def __init__( path_to_checkpoint: str = "checkpoint.pt", device: DeviceName = "cuda", device_index: int | None = None, + verbose: bool = True, ): """Initialize the Trainer. @@ -97,6 +99,8 @@ def __init__( Defaults to ``"cuda"``. device_index (int | None): The GPU device index. Only applicable when ``device="cuda"``. If ``None``, uses the default CUDA device. + verbose (bool): If True, logs epoch-level artifacts (loss, time). If False, only + start/end messages and the final training summary are printed. Defaults to True. Example: >>> from deepaudiox import AudioClassifier, Trainer @@ -114,6 +118,8 @@ def __init__( # Configure training state self.state = State() self.epochs = epochs + self.verbose = verbose + self.path_to_checkpoint = path_to_checkpoint self.device = get_device(device=device, device_index=device_index) # Configure logger @@ -137,10 +143,10 @@ def __init__( # Configure callbacks self.callbacks = [ - ConsoleLogger(logger=self.logger), Checkpointer(path_to_checkpoint=path_to_checkpoint, logger=self.logger), EarlyStopper(patience=patience, logger=self.logger), ] + self._epoch_start_time: float = 0.0 def train_step(self) -> float: """Run one pass over the training set. @@ -188,10 +194,10 @@ def val_step(self) -> float: def epoch_step(self) -> tuple[float, float]: """Run one complete training epoch. - Executes ``on_epoch_start`` callbacks, calls ``train_step()`` and - ``val_step()``, updates the LR scheduler and ``self.state``, then - executes ``on_epoch_end`` callbacks (which may trigger early stopping - or checkpointing). + Logs the epoch header and metrics when ``verbose=True``, calls ``train_step()`` + and ``val_step()``, updates the LR scheduler and + ``self.state``, then executes ``on_epoch_end`` callbacks (which may trigger + early stopping or checkpointing). Note: ``self.state.current_epoch`` must be set by the caller before @@ -220,6 +226,10 @@ def epoch_step(self) -> tuple[float, float]: ... if trainer.state.early_stop: ... break """ + self._epoch_start_time = time.time() + if self.verbose: + self.logger.info(f"[Epoch {self.state.current_epoch}/{self.epochs}]") + for cb in self.callbacks: cb.on_epoch_start(self) @@ -234,13 +244,28 @@ def epoch_step(self) -> tuple[float, float]: self.state.train_loss.append(train_loss) self.state.validation_loss.append(val_loss) + if self.verbose: + elapsed = time.time() - self._epoch_start_time + self.logger.info( + f"Epoch {self.state.current_epoch} | " + f"Train Loss: {train_loss:.4f} | " + f"Val. Loss: {val_loss:.4f} | " + f"Time: {elapsed:.2f}s" + ) + for cb in self.callbacks: cb.on_epoch_end(self) return train_loss, val_loss def train(self) -> None: - """Perform the full training process.""" + """Perform the full training process. + + Epoch-level output is controlled by ``verbose``. The training summary + (best epoch, losses, checkpoint path) is always printed on completion. + """ + self.logger.info("Training has started.") + for cb in self.callbacks: cb.on_train_start(self) @@ -253,6 +278,21 @@ def train(self) -> None: for cb in self.callbacks: cb.on_train_end(self) + best_idx = int(np.argmin(self.state.validation_loss)) + best_val_loss = self.state.validation_loss[best_idx] + best_train_loss = self.state.train_loss[best_idx] + best_epoch = best_idx + 1 + sep = "─" * 52 + self.logger.info( + f"\n{sep}\n" + f" Training Complete\n" + f" Best Epoch : {best_epoch}\n" + f" Train Loss : {best_train_loss:.6f}\n" + f" Val. Loss : {best_val_loss:.6f}\n" + f" Checkpoint : {self.path_to_checkpoint}\n" + f"{sep}" + ) + def _setup_dataloaders( self, train_dset: AudioClassificationDataset, diff --git a/uv.lock b/uv.lock index e9040b3..9095500 100644 --- a/uv.lock +++ b/uv.lock @@ -335,7 +335,7 @@ wheels = [ [[package]] name = "deepaudio-x" -version = "0.4.3" +version = "0.4.4" source = { editable = "." } dependencies = [ { name = "librosa" },