diff --git a/config.py b/config.py index 5fa2e599d..c0a6e7a18 100644 --- a/config.py +++ b/config.py @@ -68,6 +68,28 @@ MIGRATION_MAX_COLLISION_DETAILS = int(os.environ.get("MIGRATION_MAX_COLLISION_DETAILS", "1000")) TEMP_DIR = os.environ.get("TEMP_DIR", "/app/temp_audio") +# --- Direct file access --- +# When the music library is mounted into this container, analysis can read each +# track from disk instead of downloading it from the media server. Off by +# default because it needs the paths the server reports to be valid in here. +LOCAL_FILE_ACCESS = os.environ.get("LOCAL_FILE_ACCESS", "false").lower() == "true" +# Comma-separated allowlist of directories a track may be read from. REQUIRED +# when LOCAL_FILE_ACCESS is on: the path is reported by the media server, so it +# is untrusted input, and anything resolving outside these roots is refused. +LOCAL_FILE_ROOTS = os.environ.get("LOCAL_FILE_ROOTS", "") +# Rewrites the media server's path onto this container's mount point, as +# comma-separated "server_prefix=container_prefix" pairs (for example +# "/srv/music=/music"). Leave empty when the library is mounted at the same +# path the server reports. +LOCAL_FILE_PATH_MAP = os.environ.get("LOCAL_FILE_PATH_MAP", "") +# Only read from roots this process CANNOT write to, so no bug, dependency or +# future change can modify or delete a library file. Mount the library read-only +# (docker "/music:/music:ro") and leave this on; a writable root is skipped and +# its tracks are downloaded instead. Turning it off removes that guarantee. +LOCAL_FILE_REQUIRE_READONLY = os.environ.get( + "LOCAL_FILE_REQUIRE_READONLY", "true" +).lower() == "true" + def jellyfin_auth_header(token): # Jellyfin 12.0 disables the legacy X-Emby-Token header by default; the diff --git a/docs/PARAMETERS.md b/docs/PARAMETERS.md index 8ac6b1135..505a23f37 100644 --- a/docs/PARAMETERS.md +++ b/docs/PARAMETERS.md @@ -64,6 +64,10 @@ These parameters can be left as-is: | `CLEANING_CATALOGUE` | When `true`, cleaning also DELETES catalogue rows bound to no server (orphans). When `false` it only unbinds each server's stale mappings and leaves the catalogue untouched. The cleaning page has a per-run checkbox to enable it for a single run without changing this default. | `false` | | `SWEEP_PRUNE_MIN_FETCH_RATIO` | A sweep/cleaning prune is refused when the server returns fewer than this fraction of the tracks it still has mapped, so a partial fetch cannot wipe the mappings. Lower it only to prune a library that legitimately shrank that much. | `0.5` | | `MUSIC_LIBRARIES` | Comma-separated list of music libraries/folders for analysis. If empty, all libraries/folders are scanned. For Lyrion: Use folder paths like "/music/myfolder". For Navidrome/Jellyfin: Use library/folder names. | `""` (empty - scan all) | +| `LOCAL_FILE_ACCESS` | Read each track from a mounted library instead of downloading it from the media server. Needs the library mounted into the container and `LOCAL_FILE_ROOTS` set. Any path that cannot be used falls back to a normal download. | `false` | +| `LOCAL_FILE_ROOTS` | Comma-separated allowlist of directories a track may be read from. Required when `LOCAL_FILE_ACCESS` is on: the path is reported by the media server, so anything resolving outside these roots (after following symlinks) is refused. | `""` (empty - refuse everything) | +| `LOCAL_FILE_PATH_MAP` | Rewrites the media server's path onto this container's mount point, as comma-separated `server_prefix=container_prefix` pairs, longest prefix first. Example: `/srv/music=/music`. Leave empty when the library is mounted at the same path the server reports. | `""` (empty - paths used as-is) | +| `LOCAL_FILE_REQUIRE_READONLY` | Only read from roots this process cannot write to, so a library file can never be modified or deleted. Mount the library read-only (`/music:/music:ro`) and leave this on; a writable root is skipped and its tracks are downloaded instead. | `true` | | `ENABLE_PROXY_FIX` | Enable Proxy Fix for Flask when behind a reverse proxy. Example Nginx configuration: [config.py](https://github.com/NeptuneHub/AudioMuse-AI/blob/main/config.py#L918) | `false` | | `DASHBOARD_BROWSE_PAGE_SIZE` | Rows per page in the Song/Artist/Album browse view opened from the dashboard. | `100` | | `DASHBOARD_BROWSE_MAX_OFFSET` | Deepest OFFSET a browse query may reach. Past this the API stops paging and asks you to refine with search/filters, so a 1M-row catalogue cannot be hit with a pathological deep-page scan. | `50000` | diff --git a/tasks/mediaserver/__init__.py b/tasks/mediaserver/__init__.py index 5e4bf2597..a01dcf8e2 100644 --- a/tasks/mediaserver/__init__.py +++ b/tasks/mediaserver/__init__.py @@ -122,6 +122,18 @@ def get_tracks_from_album(album_id, user_creds=None, provider_type=None): def download_track(temp_dir, item): + # A mounted library is read from disk instead of fetched over HTTP. This + # returns a link inside temp_dir, never the library file, because callers + # delete what this function hands back. None means "not available locally", + # so the provider download below stays the fallback for every track. + # Imported here, not at module scope, to keep the eager import chain within + # the depth the import-architecture gate allows. + from . import local_file + + local_path = local_file.link_local_copy(temp_dir, item) + if local_path: + return local_path + provider = _provider() downloaded_path = provider.download_track(temp_dir, item) if provider is not None else None diff --git a/tasks/mediaserver/local_file.py b/tasks/mediaserver/local_file.py new file mode 100644 index 000000000..91d71a652 --- /dev/null +++ b/tasks/mediaserver/local_file.py @@ -0,0 +1,256 @@ +# AudioMuse-AI - https://github.com/NeptuneHub/AudioMuse-AI +# Copyright (C) 2025 NeptuneHub +# SPDX-License-Identifier: AGPL-3.0-only +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License v3.0. See the LICENSE file +# in the project root or + +"""Read a track from a mounted library instead of downloading it. + +Every provider reports the file path it holds a track at, and an install whose +library is mounted into the container can read that file directly: no HTTP round +trip, no second copy of the bytes, no load on the media server. This module +turns a provider's reported path into something the analysis pipeline can open, +or returns None so the caller downloads as usual. It is provider-agnostic - all +six backends populate ``Path``/``FilePath``. + +Two properties matter more than the speed: + +* The pipeline DELETES whatever ``download_track`` returns (see the ``finally`` + in tasks/analysis/album.py). Handing back the library file would delete the + user's music, so what is returned is always a symlink inside TEMP_DIR: + removing it unlinks the link and never touches the target. +* The path is reported by the media server, which makes it untrusted input. + Anything resolving outside the configured roots is refused, so a hostile or + merely wrong path cannot turn analysis into an arbitrary-file reader. + +Main Features: +* Rewrites the server's path onto the container's mount point via + LOCAL_FILE_PATH_MAP, longest prefix first, tolerating file:// URLs and + Windows separators. +* Refuses any path whose REAL location (symlinks resolved) falls outside + LOCAL_FILE_ROOTS, and refuses everything when no root is configured. +* Reads only from roots this process CANNOT write to, so the library cannot be + modified or deleted whatever the rest of the code does; a writable root is + skipped unless LOCAL_FILE_REQUIRE_READONLY is turned off. +* Publishes the track as an atomically-replaced symlink in TEMP_DIR named for + the track id, so concurrent workers cannot corrupt each other's link. +* Returns None on every failure so the caller falls back to downloading; local + access is an optimisation and never a new way for analysis to fail. +""" + +import logging +import os +import re +from urllib.parse import unquote, urlparse + +import config + +logger = logging.getLogger(__name__) + +_warned = set() +_announced = False + + +def _warn_once(key, message, *args): + if key in _warned: + return + _warned.add(key) + logger.warning(message, *args) + + +def _enabled(): + return bool(config.LOCAL_FILE_ACCESS) + + +def _roots(): + raw = config.LOCAL_FILE_ROOTS or '' + return [os.path.realpath(part.strip()) for part in raw.split(',') if part.strip()] + + +def _writable(path): + """True when THIS process could write into ``path``. + + A read-only mount fails this even for root (access(2) reports EROFS), which + is the property worth testing: not "is the mount flagged ro" but "can this + process change anything here". + """ + return os.access(path, os.W_OK) + + +def _usable_roots(): + """Roots this process may read from, dropping writable ones by default. + + Nothing here writes to the library, but a root the process CANNOT write to + means no bug, dependency or later change can delete a track either. The + check is skipped only when an operator turns LOCAL_FILE_REQUIRE_READONLY off. + """ + roots = _roots() + if not config.LOCAL_FILE_REQUIRE_READONLY: + return roots + + usable = [] + for root in roots: + if _writable(root): + _warn_once( + f'writable-root:{root}', + 'Local file access is SKIPPING %s because this process can write to ' + 'it. Mount the library read-only (for example "%s:%s:ro") so the ' + 'files cannot be modified or deleted, or set ' + 'LOCAL_FILE_REQUIRE_READONLY=false to accept the risk. Tracks under ' + 'this root are downloaded instead.', + root, root, root, + ) + continue + usable.append(root) + return usable + + +def _path_map(): + """``[(server_prefix, container_prefix)]``, longest server prefix first. + + Longest first so a specific mapping wins over a broader one covering the + same tree. + """ + raw = config.LOCAL_FILE_PATH_MAP or '' + pairs = [] + for entry in raw.split(','): + server_prefix, separator, container_prefix = entry.partition('=') + if not separator: + continue + server_prefix = server_prefix.strip().rstrip('/\\') + container_prefix = container_prefix.strip().rstrip('/') + if server_prefix and container_prefix: + pairs.append((server_prefix, container_prefix)) + pairs.sort(key=lambda pair: len(pair[0]), reverse=True) + return pairs + + +def _from_file_url(text): + if not text.lower().startswith('file://'): + return text + path = unquote(urlparse(text).path) + # file:///C:/Music/x.mp3 parses to /C:/Music/x.mp3, which is not a path on + # Windows; drop the leading slash a drive letter leaves behind. + if re.match(r'^/[A-Za-z]:', path): + path = path[1:] + return path + + +def _mapped_path(raw_path): + """The container-side path for a path the media server reported.""" + text = _from_file_url(str(raw_path or '').strip()) + if not text: + return None + for server_prefix, container_prefix in _path_map(): + if text.startswith(server_prefix): + remainder = text[len(server_prefix):].replace('\\', '/') + return container_prefix + remainder + return text + + +def _within_roots(resolved, roots): + for root in roots: + try: + if os.path.commonpath([root, resolved]) == root: + return True + except ValueError: + # Different drives (Windows) or a mix of absolute and relative. + continue + return False + + +def _resolved_local_path(raw_path): + mapped = _mapped_path(raw_path) + if not mapped: + return None + + roots = _usable_roots() + if not roots: + _warn_once( + 'no-roots', + 'LOCAL_FILE_ACCESS is enabled but no usable root is configured, so every ' + 'path is refused and tracks are downloaded instead. Set LOCAL_FILE_ROOTS ' + 'to the read-only mount the library lives on.', + ) + return None + + # realpath first: the allowlist has to be checked against where the path + # ACTUALLY lands, or a symlink inside the library would walk straight out of it. + resolved = os.path.realpath(mapped) + if not _within_roots(resolved, roots): + _warn_once( + 'outside-roots', + 'Local file access refused: %s resolves to %s, outside LOCAL_FILE_ROOTS ' + '(%s). Tracks will be downloaded. This is logged once per worker.', + mapped, resolved, ', '.join(roots), + ) + return None + + if not os.path.isfile(resolved) or not os.access(resolved, os.R_OK): + return None + try: + if os.path.getsize(resolved) <= 0: + return None + except OSError: + return None + return resolved + + +def _replace_link(target, link_path): + """Point ``link_path`` at ``target``, replacing whatever is there atomically. + + A symlink is preferred because it crosses filesystems, which matters when + TEMP_DIR is a tmpfs and the library is a mount. Windows refuses symlinks + without a privilege the service rarely holds, so a hardlink is the fallback: + equally safe here, since removing a hardlink only drops that name and the + library keeps its own. + + Two workers can be handed the same track and would build the same link name; + staging plus os.replace means neither ever sees a half-made link. + """ + staging = f"{link_path}.{os.getpid()}.tmplink" + if os.path.lexists(staging): + os.remove(staging) + try: + os.symlink(target, staging) + except (OSError, NotImplementedError): + os.link(target, staging) + os.replace(staging, link_path) + + +def link_local_copy(temp_dir, item): + """A TEMP_DIR symlink to this track's file on disk, or None to download it.""" + if not _enabled(): + return None + + resolved = _resolved_local_path(item.get('Path') or item.get('FilePath')) + if not resolved: + return None + + track_id = item.get('Id') or item.get('id') or os.path.basename(resolved) + extension = os.path.splitext(resolved)[1] or '.tmp' + link_path = os.path.join(temp_dir, f"{track_id}{extension}") + + try: + os.makedirs(temp_dir, exist_ok=True) + _replace_link(resolved, link_path) + except OSError as e: + _warn_once( + 'link-failed', + 'Could not link %s into %s (%s); falling back to downloading. This is ' + 'logged once per worker.', + resolved, temp_dir, e, + ) + return None + + global _announced + if not _announced: + _announced = True + logger.info( + 'Local file access is serving tracks from disk (first hit: %s); no ' + 'downloads are needed for files under LOCAL_FILE_ROOTS.', + resolved, + ) + return link_path diff --git a/test/unit/test_local_file_access.py b/test/unit/test_local_file_access.py new file mode 100644 index 000000000..9d364d572 --- /dev/null +++ b/test/unit/test_local_file_access.py @@ -0,0 +1,280 @@ +# AudioMuse-AI - https://github.com/NeptuneHub/AudioMuse-AI +# Copyright (C) 2025 NeptuneHub +# SPDX-License-Identifier: AGPL-3.0-only +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License v3.0. See the LICENSE file +# in the project root or + +"""Direct file access: reading a mounted library instead of downloading it. + +The two properties worth protecting are that the library file is never what the +pipeline deletes, and that a path reported by the media server cannot be used to +read outside the configured roots. Everything else is a fallback: any failure +must return None so the caller downloads the track as before. + +Main Features: +* A returned path is always a symlink in the temp dir, and deleting it (as the + analysis pipeline does) leaves the library file intact +* Paths escaping LOCAL_FILE_ROOTS are refused, including via a symlink planted + inside the library and via ../ traversal +* No roots configured, feature disabled, missing file and empty file all fall + back to downloading +* LOCAL_FILE_PATH_MAP rewrites the server's prefix onto the mount point, + longest prefix first +""" + +import os + +import pytest + + +@pytest.fixture +def library(tmp_path): + """A library root holding one track, plus a secret file outside it.""" + root = tmp_path / 'library' + (root / 'Artist' / 'Album').mkdir(parents=True) + track = root / 'Artist' / 'Album' / 'song.flac' + track.write_bytes(b'fLaC' + b'\x00' * 64) + secret = tmp_path / 'secret.txt' + secret.write_text('do not read me') + return {'root': root, 'track': track, 'secret': secret, 'tmp': tmp_path} + + +@pytest.fixture +def configured(monkeypatch, library): + from tasks.mediaserver import local_file + + monkeypatch.setattr(local_file.config, 'LOCAL_FILE_ACCESS', True, raising=False) + monkeypatch.setattr( + local_file.config, 'LOCAL_FILE_ROOTS', str(library['root']), raising=False + ) + monkeypatch.setattr(local_file.config, 'LOCAL_FILE_PATH_MAP', '', raising=False) + monkeypatch.setattr(local_file.config, 'LOCAL_FILE_REQUIRE_READONLY', True, raising=False) + # A pytest tmp dir is writable, so stand in for the read-only mount the + # feature requires. TestTheLibraryMustBeReadOnly checks the real primitive. + monkeypatch.setattr(local_file, '_writable', lambda path: False) + local_file._warned.clear() + return local_file + + +def _temp_dir(library): + path = library['tmp'] / 'temp_audio' + path.mkdir(exist_ok=True) + return str(path) + + +class TestTheLibraryFileIsNeverAtRisk: + def test_the_returned_path_is_a_link_in_the_temp_dir(self, configured, library): + link = configured.link_local_copy( + _temp_dir(library), {'Id': '42', 'Path': str(library['track'])} + ) + + assert link is not None + assert os.path.dirname(link) == _temp_dir(library) + assert os.path.samefile(link, str(library['track'])) + + def test_deleting_the_returned_path_leaves_the_library_untouched(self, configured, library): + """The analysis pipeline removes what download_track returns.""" + link = configured.link_local_copy( + _temp_dir(library), {'Id': '42', 'Path': str(library['track'])} + ) + + os.remove(link) + + assert not os.path.lexists(link) + assert library['track'].exists() + assert library['track'].read_bytes().startswith(b'fLaC') + + def test_the_link_keeps_the_real_extension(self, configured, library): + link = configured.link_local_copy( + _temp_dir(library), {'Id': '42', 'Path': str(library['track'])} + ) + + assert os.path.basename(link) == '42.flac' + + def test_a_second_call_replaces_the_link_rather_than_failing(self, configured, library): + item = {'Id': '42', 'Path': str(library['track'])} + first = configured.link_local_copy(_temp_dir(library), item) + second = configured.link_local_copy(_temp_dir(library), item) + + assert first == second + assert os.path.samefile(second, str(library['track'])) + + +class TestTheLibraryMustBeReadOnly: + """Nothing here writes to the library; a read-only root means nothing CAN.""" + + def test_the_writable_check_is_real(self, library): + """Guards the simulation in the fixture from making the suite vacuous.""" + from tasks.mediaserver import local_file + + assert local_file._writable(str(library['root'])) is True + + def test_a_writable_root_is_skipped_by_default(self, configured, library, monkeypatch): + monkeypatch.setattr(configured, '_writable', lambda path: True) + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(library['track'])} + ) is None + + def test_the_requirement_can_be_waived(self, configured, library, monkeypatch): + monkeypatch.setattr(configured, '_writable', lambda path: True) + monkeypatch.setattr( + configured.config, 'LOCAL_FILE_REQUIRE_READONLY', False, raising=False + ) + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(library['track'])} + ) is not None + + +class TestUntrustedPathsAreRefused: + def test_a_path_outside_the_roots_is_refused(self, configured, library): + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(library['secret'])} + ) is None + + def test_traversal_out_of_the_root_is_refused(self, configured, library): + escape = str(library['root'] / 'Artist' / '..' / '..' / 'secret.txt') + + assert configured.link_local_copy(_temp_dir(library), {'Id': '1', 'Path': escape}) is None + + def test_a_symlink_inside_the_library_cannot_point_out_of_it(self, configured, library): + """realpath is checked, not the path as given.""" + planted = library['root'] / 'Artist' / 'escape.flac' + try: + os.symlink(str(library['secret']), str(planted)) + except (OSError, NotImplementedError): + pytest.skip('symlink creation is not permitted in this environment') + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(planted)} + ) is None + + def test_no_configured_root_refuses_everything(self, configured, library, monkeypatch): + monkeypatch.setattr(configured.config, 'LOCAL_FILE_ROOTS', '', raising=False) + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(library['track'])} + ) is None + + +class TestFallingBackToDownloading: + def test_disabled_returns_none_without_touching_the_path(self, configured, library, monkeypatch): + monkeypatch.setattr(configured.config, 'LOCAL_FILE_ACCESS', False, raising=False) + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(library['track'])} + ) is None + + def test_a_missing_file_returns_none(self, configured, library): + missing = str(library['root'] / 'Artist' / 'Album' / 'gone.flac') + + assert configured.link_local_copy(_temp_dir(library), {'Id': '1', 'Path': missing}) is None + + def test_an_empty_file_returns_none(self, configured, library): + empty = library['root'] / 'Artist' / 'Album' / 'empty.flac' + empty.write_bytes(b'') + + assert configured.link_local_copy( + _temp_dir(library), {'Id': '1', 'Path': str(empty)} + ) is None + + def test_an_item_with_no_path_returns_none(self, configured, library): + assert configured.link_local_copy(_temp_dir(library), {'Id': '1'}) is None + + +class TestPathMapping: + def test_the_server_prefix_is_rewritten_onto_the_mount_point( + self, configured, library, monkeypatch + ): + monkeypatch.setattr( + configured.config, + 'LOCAL_FILE_PATH_MAP', + f"/srv/music={library['root']}", + raising=False, + ) + + link = configured.link_local_copy( + _temp_dir(library), {'Id': '7', 'Path': '/srv/music/Artist/Album/song.flac'} + ) + + assert link is not None + assert os.path.samefile(link, str(library['track'])) + + def test_the_longest_matching_prefix_wins(self, configured, library, monkeypatch): + monkeypatch.setattr( + configured.config, + 'LOCAL_FILE_PATH_MAP', + f"/srv={library['tmp'] / 'wrong'},/srv/music={library['root']}", + raising=False, + ) + + link = configured.link_local_copy( + _temp_dir(library), {'Id': '7', 'Path': '/srv/music/Artist/Album/song.flac'} + ) + + assert link is not None + assert os.path.samefile(link, str(library['track'])) + + def test_a_file_url_is_understood(self, configured, library): + url = 'file://' + str(library['track']).replace(os.sep, '/') + if not url.startswith('file:///'): + url = url.replace('file://', 'file:///', 1) + + link = configured.link_local_copy(_temp_dir(library), {'Id': '7', 'Path': url}) + + assert link is not None + assert os.path.samefile(link, str(library['track'])) + + def test_windows_separators_survive_the_rewrite(self, configured, library, monkeypatch): + monkeypatch.setattr( + configured.config, + 'LOCAL_FILE_PATH_MAP', + f"D:\\Media={library['root']}", + raising=False, + ) + + link = configured.link_local_copy( + _temp_dir(library), {'Id': '7', 'Path': 'D:\\Media\\Artist\\Album\\song.flac'} + ) + + assert link is not None + assert os.path.samefile(link, str(library['track'])) + + +class TestDispatcherIntegration: + def test_a_local_hit_never_calls_the_provider(self, configured, library, monkeypatch): + from unittest.mock import MagicMock + + from tasks import mediaserver + + provider = MagicMock() + monkeypatch.setattr(mediaserver, '_provider', lambda *a, **k: provider) + + path = mediaserver.download_track( + _temp_dir(library), {'Id': '42', 'Path': str(library['track'])} + ) + + assert os.path.samefile(path, str(library['track'])) + provider.download_track.assert_not_called() + + def test_a_local_miss_falls_through_to_the_provider(self, configured, library, monkeypatch): + from unittest.mock import MagicMock + + from tasks import mediaserver + + downloaded = os.path.join(_temp_dir(library), 'downloaded.mp3') + with open(downloaded, 'wb') as handle: + handle.write(b'ID3') + provider = MagicMock() + provider.download_track.return_value = downloaded + monkeypatch.setattr(mediaserver, '_provider', lambda *a, **k: provider) + + path = mediaserver.download_track( + _temp_dir(library), {'Id': '42', 'Path': str(library['secret'])} + ) + + assert path == downloaded + provider.download_track.assert_called_once()