diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9efe08b..7c35c04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: run: python -m pip install customtkinter darkdetect Pillow - name: Byte-compile all sources - run: python -m compileall -q main.py app.py scanner.py file_utils.py analysis.py treemap_render.py thumbnails.py annotate.py imagenav.py updater.py version.py registry_installer.py simple_installer.py make_version_info.py + run: python -m compileall -q main.py app.py scanner.py file_utils.py analysis.py treemap_render.py thumbnails.py annotate.py imagenav.py duplicates.py trash.py updater.py version.py registry_installer.py simple_installer.py make_version_info.py - name: Validate version resource generation run: python make_version_info.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0435bf5..4e0a059 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,6 +56,9 @@ jobs: run: | $tag = if ($env:INPUT_TAG) { $env:INPUT_TAG } else { "${{ github.ref_name }}" } $ver = $tag -replace '^v', '' + # a build-only dispatch runs on a branch, whose name can contain '/' + # and other characters that are not legal in a file name + $ver = $ver -replace '[^0-9A-Za-z._-]', '-' if (-not $ver) { $ver = "dev" } Compress-Archive -Path dist_onedir\FolderLens\* -DestinationPath "FolderLens-$ver-win64.zip" -Force Get-ChildItem FolderLens-*.zip diff --git a/README.md b/README.md index 2bfd7db..1fd7b17 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ # FolderLens **A fast, modern folder size analyzer for Windows.** -See what's eating your disk — as a tree, a treemap, a top-files list, or a file-type breakdown. +See what's eating your disk — as a tree, a treemap, a top-files list, a file-type breakdown, or duplicate copies. [![CI](https://github.com/MrHakan/FolderLens/actions/workflows/ci.yml/badge.svg)](https://github.com/MrHakan/FolderLens/actions/workflows/ci.yml) [![Release](https://github.com/MrHakan/FolderLens/actions/workflows/release.yml/badge.svg)](https://github.com/MrHakan/FolderLens/actions/workflows/release.yml) @@ -22,13 +22,14 @@ Grab the latest build from the [Releases page](https://github.com/MrHakan/Folder ## Features FolderLens scans a whole directory tree **once** — in parallel, across up to 32 -worker threads — then lets you explore it four different ways with zero +worker threads — then lets you explore it five different ways with zero rescanning: - 🌳 **Tree view** — expandable folder tree with a usage bar, size, item count, type, and date at every level. Expanding a folder is instant. - 🗺️ **Treemap** — cushion-shaded map where every rectangle's area is its size, folders get their own header band, and **image files are painted with their own thumbnail** so you can recognise them at a glance. Hover for a **peek preview** of the picture, click a folder to zoom in, right-click to go back. - 🏆 **Largest files** — the top 100 biggest files anywhere in the tree, with their locations and small inline previews. - 🧩 **File types** — size and count broken down by category (video, image, code, …) with proportional bars. +- 👯 **Duplicates** — finds byte-identical copies and shows exactly how much space keeping one of each would free. Narrowed by size, then a head/tail sample, then a full hash, so almost nothing is read twice. ### Image viewer & annotation @@ -48,7 +49,8 @@ Plus: - 🔎 **Instant search** across the whole tree (Ctrl+F) - 🧵 **Fully responsive** — scanning, zipping, deleting, and exporting all run off the UI thread, with live progress and a **Stop** button - 🗑️ **Manage** — multi-select to zip, delete, or open in Explorer (right-click, toolbar, or Delete key); sizes update without rescanning -- 📤 **Export** the full report to CSV +- 📤 **Export** the full report to CSV, or the treemap itself as a PNG +- ♻️ **Recycle Bin** — deletions are undoable by default (permanent delete is a setting) - 💽 **Disk usage** shown in the status bar (free / total) - 🌗 **Light / dark** theme, remembered between sessions, along with your last folder and view - ⬆️ **Auto-update** from GitHub releases @@ -82,7 +84,7 @@ python main.py --version | `F5` | Rescan current folder | | `Ctrl+F` | Focus search | | `Esc` | Clear search | -| `Delete` | Delete selected (tree / largest files) | +| `Delete` | Delete selected (tree / largest files / duplicates) | | Double-click | Open folder / open image viewer | In the image viewer: @@ -127,6 +129,8 @@ FolderLens/ ├── app.py # UI (customtkinter + ttk): views, image viewer, toolbars ├── scanner.py # single-pass parallel tree scanner ├── analysis.py # treemap layout, largest-files, type breakdown, CSV (pure, tested) +├── duplicates.py # size -> sample -> full hash duplicate detection (pure, tested) +├── trash.py # Recycle Bin / XDG trash, with a permanent-delete fallback ├── treemap_render.py # cushion shading, thumbnails, labels, hit-testing ├── thumbnails.py # background thumbnail decoding + LRU cache ├── annotate.py # annotation model, tools, undo/redo, export (pure, tested) diff --git a/app.py b/app.py index 0958ad1..58adeba 100644 --- a/app.py +++ b/app.py @@ -20,8 +20,10 @@ from scanner import TreeScanner, Node import analysis import annotate +import duplicates import imagenav import treemap_render +import trash from thumbnails import ThumbnailCache, fit_box from version import VERSION from updater import get_updater @@ -61,6 +63,7 @@ def __init__(self): self.list_thumbnails = True self.peek_preview = True self.annotation_mode = "Basic" + self.use_recycle_bin = True self.load() def row_height(self) -> int: @@ -81,9 +84,10 @@ def load(self): self.dark_mode = data['dark_mode'] if isinstance(data.get('last_folder'), str): self.last_folder = data['last_folder'] - if data.get('view') in ("Tree", "Treemap", "Largest Files", "File Types"): + if data.get('view') in ("Tree", "Treemap", "Largest Files", "File Types", "Duplicates"): self.view = data['view'] - for flag in ('treemap_thumbnails', 'list_thumbnails', 'peek_preview'): + for flag in ('treemap_thumbnails', 'list_thumbnails', 'peek_preview', + 'use_recycle_bin'): if isinstance(data.get(flag), bool): setattr(self, flag, data[flag]) if data.get('annotation_mode') in ("Basic", "Advanced"): @@ -106,6 +110,7 @@ def save(self): 'list_thumbnails': self.list_thumbnails, 'peek_preview': self.peek_preview, 'annotation_mode': self.annotation_mode, + 'use_recycle_bin': self.use_recycle_bin, }, f, indent=2) except OSError: pass @@ -691,6 +696,11 @@ def __init__(self, master, settings: AppSettings, on_apply, **kwargs): ctk.CTkSwitch(main, text="Show small previews in lists", variable=self.list_thumbs_var).pack(anchor="w", pady=6) + ctk.CTkLabel(main, text="Deleting", font=ctk.CTkFont(size=14, weight="bold")).pack(anchor="w", pady=(12, 0)) + self.recycle_var = ctk.BooleanVar(value=self.settings.use_recycle_bin) + ctk.CTkSwitch(main, text="Send deleted files to the Recycle Bin", + variable=self.recycle_var).pack(anchor="w", pady=6) + ctk.CTkLabel(main, text="Annotation", font=ctk.CTkFont(size=14, weight="bold")).pack(anchor="w", pady=(12, 0)) ctk.CTkLabel(main, text="Basic: pen, marker, arrow, eraser.\nAdvanced: adds shapes, text, redo.", font=ctk.CTkFont(size=11), text_color="gray", @@ -706,6 +716,7 @@ def _apply(self): self.settings.treemap_thumbnails = self.treemap_thumbs_var.get() self.settings.list_thumbnails = self.list_thumbs_var.get() self.settings.annotation_mode = self.annotation_var.get() + self.settings.use_recycle_bin = self.recycle_var.get() self.on_apply() self.destroy() @@ -811,7 +822,7 @@ class FolderLensApp(ctk.CTk): """Fast, multi-view folder size explorer.""" BAR_WIDTH = 10 - VIEWS = ["Tree", "Treemap", "Largest Files", "File Types"] + VIEWS = ["Tree", "Treemap", "Largest Files", "File Types", "Duplicates"] def __init__(self, initial_path: Optional[str] = None): super().__init__() @@ -841,11 +852,20 @@ def __init__(self, initial_path: Optional[str] = None): self.largest_tree: Optional[ttk.Treeview] = None self.largest_map: Dict[str, Node] = {} + # duplicates view state + self.dup_tree: Optional[ttk.Treeview] = None + self.dup_map: Dict[str, Node] = {} + self.dup_groups: List[duplicates.DuplicateGroup] = [] + self._dup_running = False + self._dup_cancel = False + # treemap state self.treemap_stack: List[Node] = [] self._tiles: List[analysis.Tile] = [] self._hover_tile = None + self._highlight_id = None self._treemap_photo = None + self._treemap_image = None self._peek_path: Optional[str] = None self._peek_args = None self.tooltip: Optional[Tooltip] = None @@ -893,6 +913,13 @@ def _colors(self) -> dict: # which is why buttons went missing on smaller windows. NARROW_WIDTH = 1120 + # Tk keeps every PhotoImage alive for as long as we reference it, so the + # row-preview map is capped rather than left to grow while browsing. + MAX_ROW_PHOTOS = 400 + + # duplicates below this are rarely worth the read time + DUPLICATE_MIN_SIZE = 4096 + def _build_toolbar(self): self.toolbar = ctk.CTkFrame(self, fg_color=("gray95", "gray14"), corner_radius=0) self.toolbar.pack(fill="x") @@ -945,9 +972,10 @@ def _build_toolbar(self): text_color=("gray30", "gray70"), hover_color=("gray85", "gray25"), command=self._toggle_theme) self.theme_btn.pack(side="right", padx=4) - ctk.CTkButton(self.actions, text="⬇ CSV", width=70, height=34, font=ctk.CTkFont(size=12), - fg_color="transparent", border_width=1, text_color=("gray20", "gray80"), - command=self._export_csv).pack(side="right", padx=4) + ctk.CTkButton(self.actions, text="⬇ Export", width=84, height=34, + font=ctk.CTkFont(size=12), fg_color="transparent", border_width=1, + text_color=("gray20", "gray80"), + command=self._show_export_menu).pack(side="right", padx=4) self.search_var = ctk.StringVar() self.search_entry = ctk.CTkEntry(self.toolbar, textvariable=self.search_var, @@ -1108,6 +1136,12 @@ def _clear_body(self): # reaches for a stale one later self.tree = None self.largest_tree = None + self.dup_tree = None + # every row that was pointing at a thumbnail is gone with them; the + # map would otherwise grow for the lifetime of the session + self._row_by_path.clear() + if len(self._row_photos) > self.MAX_ROW_PHOTOS: + self._row_photos.clear() def _render_active_view(self): self._clear_body() @@ -1119,6 +1153,8 @@ def _render_active_view(self): self._render_largest() elif self.active_view == "File Types": self._render_types() + elif self.active_view == "Duplicates": + self._render_duplicates() def _empty_hint(self, text: str): wrap = tk.Frame(self.body, bg=self._colors()['tree_bg']) @@ -1447,6 +1483,149 @@ def _render_types(self): fill = tk.Frame(track, bg=stat.color, height=14) fill.place(relx=0, rely=0, relwidth=max(stat.size / total, 0.004), relheight=1) + # ---- Duplicates view + + def _render_duplicates(self): + if not self.root_node: + self._empty_hint("Select a folder to analyze") + return + + colors = self._colors() + bar = tk.Frame(self.body, bg=colors['head_bg'], height=38) + bar.pack(fill="x") + bar.pack_propagate(False) + + self.dup_status = tk.Label(bar, text="", bg=colors['head_bg'], fg=colors['head_fg'], + font=("Segoe UI", 10)) + self.dup_status.pack(side="left", padx=12) + + self.dup_button = tk.Button(bar, text="Find duplicates", bd=0, relief="flat", + cursor="hand2", bg=ACCENT, fg="white", + activebackground=ACCENT_HOVER, activeforeground="white", + font=("Segoe UI", 10, "bold"), padx=14, pady=4, + command=self._toggle_duplicate_scan) + self.dup_button.pack(side="right", padx=10, pady=6) + + headings = { + "#0": ("File / group", lambda: None), + "size": ("Size", lambda: None), + "wasted": ("Reclaimable", lambda: None), + "path": ("Location", lambda: None), + } + widths = { + "#0": (360, 220, "w", False), + "size": (100, 80, "e", False), + "wasted": (110, 90, "e", False), + "path": (520, 240, "w", True), + } + self.dup_tree = self._make_treeview(("size", "wasted", "path"), headings, widths) + self.dup_map = {} + self.dup_tree.bind("<>", self._on_tree_select) + self.dup_tree.bind("", lambda e: self._delete_selected()) + self.dup_tree.bind("", self._on_duplicate_double) + + if self.dup_groups: + self._fill_duplicates() + else: + self.dup_status.configure( + text="Find byte-identical copies and reclaim the space they waste") + + def _on_duplicate_double(self, event): + node = self.dup_map.get(self.dup_tree.identify_row(event.y)) + if node: + self._reveal(node.path) + + def _toggle_duplicate_scan(self): + if self._dup_running: + self._dup_cancel = True + return + self._start_duplicate_scan() + + def _start_duplicate_scan(self): + root = self.root_node + if not root: + return + self._dup_running = True + self._dup_cancel = False + self.dup_button.configure(text="Stop") + self.dup_status.configure(text="Scanning…") + + def report(stage, done, total): + if total: + self.after(0, lambda: self._safe_dup_status(f"{stage}… {done:,}/{total:,}")) + + def worker(): + try: + found = duplicates.find_duplicates( + root, min_size=self.DUPLICATE_MIN_SIZE, + progress=report, should_cancel=lambda: self._dup_cancel) + except Exception as exc: + message = str(exc) + self.after(0, lambda: self._duplicate_scan_failed(message)) + return + self.after(0, lambda: self._duplicate_scan_done(found)) + + threading.Thread(target=worker, daemon=True).start() + + def _safe_dup_status(self, text: str): + if self.active_view == "Duplicates" and getattr(self, "dup_status", None) is not None: + try: + self.dup_status.configure(text=text) + except tk.TclError: + pass + + def _duplicate_scan_failed(self, message: str): + self._dup_running = False + self._safe_dup_status(f"Failed: {message}") + if self.active_view == "Duplicates": + try: + self.dup_button.configure(text="Find duplicates") + except tk.TclError: + pass + + def _duplicate_scan_done(self, groups): + self._dup_running = False + cancelled = self._dup_cancel + self.dup_groups = groups + if self.active_view != "Duplicates": + return + try: + self.dup_button.configure(text="Rescan") + except tk.TclError: + return + if cancelled: + self._safe_dup_status("Cancelled") + return + self._fill_duplicates() + + def _fill_duplicates(self): + tree = self.dup_tree + if tree is None: + return + tree.delete(*tree.get_children()) + self.dup_map = {} + + if not self.dup_groups: + self._safe_dup_status("No duplicate files found") + return + + reclaimable = duplicates.total_wasted(self.dup_groups) + self._safe_dup_status( + f"{len(self.dup_groups):,} groups · {format_size(reclaimable)} reclaimable") + + for group in self.dup_groups: + parent = tree.insert( + "", "end", + text=f"{ICONS['folder_open']} {group.count} copies · {group.nodes[0].name}", + values=(format_size(group.size), format_size(group.wasted), ""), + tags=("folder",), open=False) + for node in sorted(group.nodes, key=lambda n: (len(n.path), n.path)): + iid = tree.insert( + parent, "end", text=f"{get_file_icon(node.path)} {node.name}", + values=(format_size(node.size), "", os.path.dirname(node.path))) + self.dup_map[iid] = node + self._register_row_thumbnail(tree, iid, node) + # ---- Treemap view def _render_treemap(self): @@ -1487,7 +1666,7 @@ def _treemap_leave(self, event): self.tooltip.hide() if self._hover_tile is not None: self._hover_tile = None - self._schedule_treemap_redraw() + self._draw_highlight(None) def _schedule_treemap_redraw(self): """Coalesce the burst of events a window resize produces @@ -1529,16 +1708,33 @@ def _draw_treemap(self, highlight=None): opts = treemap_render.RenderOptions( dark_mode=self.settings.dark_mode, show_thumbnails=self.settings.treemap_thumbnails, - highlight=highlight, ) image = treemap_render.render_treemap( self._tiles, w, h, opts, thumb_provider=self._treemap_thumb if self.settings.treemap_thumbnails else None) # one canvas image instead of thousands of items: far less work for Tk + self._treemap_image = image self._treemap_photo = ImageTk.PhotoImage(image) canvas.delete("all") canvas.create_image(0, 0, anchor="nw", image=self._treemap_photo) + self._highlight_id = None + if highlight is not None: + self._draw_highlight(highlight) + + def _draw_highlight(self, tile): + """Outline the hovered tile as a canvas item on top of the rendered + image. Re-rendering the whole map just to move this outline cost a + full re-composite of every tile on each mouse move.""" + canvas = self.treemap_canvas + if self._highlight_id is not None: + canvas.delete(self._highlight_id) + self._highlight_id = None + if tile is None: + return + self._highlight_id = canvas.create_rectangle( + tile.x, tile.y, tile.x + tile.w - 1, tile.y + tile.h - 1, + outline="#ffffff", width=2) def _treemap_thumb(self, path: str, size): return self.thumbnails.request(path, (min(size[0], 400), min(size[1], 400))) @@ -1566,12 +1762,12 @@ def _treemap_hover(self, event): self.tooltip.hide() if self._hover_tile is not None: self._hover_tile = None - self._schedule_treemap_redraw() + self._draw_highlight(None) return if tile is not self._hover_tile: self._hover_tile = tile - self._draw_treemap(highlight=tile) + self._draw_highlight(tile) n = tile.node kind = "Folder" if n.is_dir else get_file_category(n.path)['label'] @@ -1641,6 +1837,8 @@ def _selection_context(self): return self.tree, self.iid_to_node if self.active_view == "Largest Files" and self.largest_tree is not None: return self.largest_tree, self.largest_map + if self.active_view == "Duplicates" and self.dup_tree is not None: + return self.dup_tree, self.dup_map return None, {} def _selected_nodes(self) -> List[Node]: @@ -1689,6 +1887,34 @@ def _open_in_explorer(self): if nodes: self._reveal(nodes[0].path) + def _show_export_menu(self): + menu = tk.Menu(self, tearoff=0) + menu.add_command(label="Report as CSV…", command=self._export_csv) + menu.add_command(label="Treemap as PNG…", command=self._export_treemap) + try: + x = self.winfo_pointerx() + y = self.winfo_pointery() + menu.tk_popup(x, y) + finally: + menu.grab_release() + + def _export_treemap(self): + image = getattr(self, "_treemap_image", None) + if image is None: + messagebox.showinfo("Nothing to export", + "Open the Treemap view first.", parent=self) + return + target = filedialog.asksaveasfilename( + defaultextension=".png", filetypes=[("PNG image", "*.png")], + initialfile="folderlens-treemap.png", title="Save treemap image as") + if not target: + return + try: + image.save(target) + self._set_status(f"Saved {os.path.basename(target)}") + except Exception as exc: + messagebox.showerror("Export failed", str(exc)) + def _export_csv(self): if not self.root_node: messagebox.showwarning("No data", "Scan a folder first.") @@ -1745,9 +1971,9 @@ def worker(): threading.Thread(target=worker, daemon=True).start() def _no_selection_hint(self) -> str: - if self.active_view in ("Tree", "Largest Files"): + if self.active_view in ("Tree", "Largest Files", "Duplicates"): return "Select files or folders first." - return "Switch to the Tree or Largest Files view to select items." + return "Switch to the Tree, Largest Files or Duplicates view to select items." def _delete_selected(self): selection = self._top_level_selection() @@ -1755,10 +1981,16 @@ def _delete_selected(self): messagebox.showwarning("No selection", self._no_selection_hint()) return total = sum(node.size for _, node in selection) - if not messagebox.askyesno("Confirm delete", - f"Delete {len(selection)} item(s) ({format_size(total)})?\nThis cannot be undone."): + recycle = self.settings.use_recycle_bin and trash.is_supported() + if recycle: + prompt = (f"Move {len(selection)} item(s) ({format_size(total)}) to the " + f"Recycle Bin?\nYou can restore them from there.") + else: + prompt = (f"Delete {len(selection)} item(s) ({format_size(total)})?\n" + f"This cannot be undone.") + if not messagebox.askyesno("Confirm delete", prompt): return - self._set_status("Deleting…") + self._set_status("Recycling…" if recycle else "Deleting…") # remember which view started this so the async result never touches a # widget the user has since navigated away from @@ -1768,7 +2000,12 @@ def worker(): deleted, errors = [], [] for iid, node in selection: try: - if os.path.isdir(node.path): + if recycle: + moved, message = trash.send_to_trash(node.path) + if not moved: + errors.append(f"{node.name}: {message}") + continue + elif os.path.isdir(node.path): shutil.rmtree(node.path) else: os.remove(node.path) diff --git a/duplicates.py b/duplicates.py new file mode 100644 index 0000000..1e9b1f3 --- /dev/null +++ b/duplicates.py @@ -0,0 +1,134 @@ +"""Duplicate file detection over an already-scanned tree. + +Hashing every file would be far too slow on a real drive, so this narrows the +candidates in three stages, each cheaper than the one after it: + + 1. group by exact size - no I/O at all, and most files are unique by size + 2. hash a small head/tail sample - one short read, splits nearly all the rest + 3. hash the full contents - only for files that still look identical + +Pure logic with injectable progress/cancel, so it is testable without a UI. +""" +import hashlib +import os +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +SAMPLE_BYTES = 65536 # head+tail sample used for the cheap pass +CHUNK = 1 << 20 # 1 MiB streaming reads for the full hash + + +@dataclass +class DuplicateGroup: + size: int + nodes: List[object] = field(default_factory=list) + + @property + def count(self) -> int: + return len(self.nodes) + + @property + def wasted(self) -> int: + """Space that would be freed by keeping a single copy.""" + return self.size * max(0, self.count - 1) + + +def sample_digest(path: str, sample: int = SAMPLE_BYTES) -> Optional[str]: + """Hash of the first and last `sample` bytes, plus the size. + + Cheap enough to run on every same-size candidate, and files that differ + almost always differ near one end. + """ + try: + size = os.path.getsize(path) + h = hashlib.blake2b(digest_size=16) + h.update(str(size).encode()) + with open(path, "rb") as f: + h.update(f.read(sample)) + if size > sample * 2: + f.seek(-sample, os.SEEK_END) + h.update(f.read(sample)) + return h.hexdigest() + except OSError: + return None + + +def full_digest(path: str) -> Optional[str]: + try: + h = hashlib.blake2b(digest_size=16) + with open(path, "rb") as f: + for block in iter(lambda: f.read(CHUNK), b""): + h.update(block) + return h.hexdigest() + except OSError: + return None + + +def _group_by(items, key_func, should_cancel) -> List[List]: + """Bucket items by a key, keeping only buckets with more than one member.""" + buckets: Dict[object, List] = {} + for item in items: + if should_cancel and should_cancel(): + return [] + key = key_func(item) + if key is None: # unreadable: cannot be proven a duplicate + continue + buckets.setdefault(key, []).append(item) + return [group for group in buckets.values() if len(group) > 1] + + +def find_duplicates(root, min_size: int = 1, + progress: Optional[Callable[[str, int, int], None]] = None, + should_cancel: Optional[Callable[[], bool]] = None + ) -> List[DuplicateGroup]: + """Find groups of byte-identical files, biggest waste first. + + `min_size` skips small files, where duplicates are common and reclaiming + them is not worth the read. + """ + from analysis import iter_file_nodes + + files = [n for n in iter_file_nodes(root) if n.size >= min_size] + if progress: + progress("Grouping by size", 0, len(files)) + + by_size = _group_by(files, lambda n: n.size, should_cancel) + if should_cancel and should_cancel(): + return [] + + candidates = [n for group in by_size for n in group] + if progress: + progress("Sampling candidates", 0, len(candidates)) + + groups: List[DuplicateGroup] = [] + done = 0 + for same_size in by_size: + if should_cancel and should_cancel(): + return [] + + for sampled in _group_by(same_size, lambda n: sample_digest(n.path), should_cancel): + if should_cancel and should_cancel(): + return [] + # a full hash is only needed when the sample already matched + for identical in _group_by(sampled, lambda n: full_digest(n.path), should_cancel): + groups.append(DuplicateGroup(size=identical[0].size, nodes=identical)) + + done += len(same_size) + if progress: + progress("Hashing", done, len(candidates)) + + groups.sort(key=lambda g: g.wasted, reverse=True) + return groups + + +def total_wasted(groups: List[DuplicateGroup]) -> int: + return sum(g.wasted for g in groups) + + +def keep_first_delete_rest(group: DuplicateGroup) -> List[object]: + """The copies that can go: everything but the shortest path. + + The shortest path is usually the original rather than a 'copy (2)' of it. + """ + ordered = sorted(group.nodes, key=lambda n: (len(n.path), n.path)) + return ordered[1:] diff --git a/scanner.py b/scanner.py index 24ad943..4b8c053 100644 --- a/scanner.py +++ b/scanner.py @@ -1,23 +1,47 @@ import os import threading from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import List, Callable, Optional import time -@dataclass +_NO_CHILDREN: tuple = () + + class Node: - """One file or directory in the scanned tree.""" - path: str - name: str - is_dir: bool - size: int = 0 - creation_date: float = 0.0 - item_count: int = 0 - children: List["Node"] = field(default_factory=list) - parent: Optional["Node"] = None - error: Optional[str] = None + """One file or directory in the scanned tree. + + Uses __slots__ rather than a dataclass: a scan of a large drive holds one + of these per file, and dropping the per-instance __dict__ cuts the tree's + memory footprint by roughly a third. + """ + + __slots__ = ("path", "name", "is_dir", "size", "creation_date", + "item_count", "children", "parent", "error") + + def __init__(self, path: str, name: str, is_dir: bool, size: int = 0, + creation_date: float = 0.0, item_count: int = 0, + children: Optional[List["Node"]] = None, + parent: Optional["Node"] = None, error: Optional[str] = None): + self.path = path + self.name = name + self.is_dir = is_dir + self.size = size + self.creation_date = creation_date + self.item_count = item_count + if children is not None: + self.children = children + else: + # files can never have children, so they share one empty tuple + # instead of each allocating a list they will never use + self.children = [] if is_dir else _NO_CHILDREN + self.parent = parent + self.error = error + + def __repr__(self) -> str: + kind = "dir" if self.is_dir else "file" + return f"" def sorted_children(self, key: str = "size", reverse: bool = True) -> List["Node"]: if key == "name": diff --git a/tests/test_duplicates.py b/tests/test_duplicates.py new file mode 100644 index 0000000..4334d69 --- /dev/null +++ b/tests/test_duplicates.py @@ -0,0 +1,133 @@ +import os +import sys +import threading + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import duplicates +from scanner import TreeScanner + + +def scan(path): + holder, done = {}, threading.Event() + TreeScanner().scan(str(path), + on_complete=lambda r, e, t: (holder.update(root=r), done.set()), + on_error=lambda m: done.set()) + assert done.wait(30) + return holder["root"] + + +@pytest.fixture +def tree(tmp_path): + """ + twins: a.bin == b.bin == nested/c.bin (identical) + same size but different content: x.bin vs y.bin + unique: solo.bin + """ + payload = b"D" * 9000 + (tmp_path / "a.bin").write_bytes(payload) + (tmp_path / "b.bin").write_bytes(payload) + nested = tmp_path / "nested" + nested.mkdir() + (nested / "c.bin").write_bytes(payload) + + (tmp_path / "x.bin").write_bytes(b"X" * 7000) + (tmp_path / "y.bin").write_bytes(b"Y" * 7000) # same size, different bytes + (tmp_path / "solo.bin").write_bytes(b"S" * 5000) + return tmp_path + + +def test_finds_identical_files(tree): + groups = duplicates.find_duplicates(scan(tree), min_size=1) + assert len(groups) == 1 + + group = groups[0] + assert group.count == 3 + assert {n.name for n in group.nodes} == {"a.bin", "b.bin", "c.bin"} + assert group.size == 9000 + assert group.wasted == 18000 # two redundant copies + + +def test_same_size_different_content_is_not_a_duplicate(tree): + groups = duplicates.find_duplicates(scan(tree), min_size=1) + names = {n.name for g in groups for n in g.nodes} + assert "x.bin" not in names and "y.bin" not in names + + +def test_unique_files_are_ignored(tree): + groups = duplicates.find_duplicates(scan(tree), min_size=1) + assert "solo.bin" not in {n.name for g in groups for n in g.nodes} + + +def test_min_size_skips_small_files(tree): + assert duplicates.find_duplicates(scan(tree), min_size=100_000) == [] + + +def test_groups_sorted_by_reclaimable_space(tmp_path): + (tmp_path / "small1.bin").write_bytes(b"a" * 1000) + (tmp_path / "small2.bin").write_bytes(b"a" * 1000) + (tmp_path / "big1.bin").write_bytes(b"b" * 50000) + (tmp_path / "big2.bin").write_bytes(b"b" * 50000) + + groups = duplicates.find_duplicates(scan(tmp_path), min_size=1) + assert [g.size for g in groups] == [50000, 1000] + assert duplicates.total_wasted(groups) == 51000 + + +def test_cancellation_returns_nothing(tree): + groups = duplicates.find_duplicates(scan(tree), min_size=1, + should_cancel=lambda: True) + assert groups == [] + + +def test_progress_is_reported(tree): + seen = [] + duplicates.find_duplicates(scan(tree), min_size=1, + progress=lambda stage, done, total: seen.append(stage)) + assert seen + + +def test_empty_tree(tmp_path): + assert duplicates.find_duplicates(scan(tmp_path), min_size=1) == [] + + +def test_keep_first_delete_rest_keeps_the_shortest_path(tree): + group = duplicates.find_duplicates(scan(tree), min_size=1)[0] + doomed = duplicates.keep_first_delete_rest(group) + assert len(doomed) == group.count - 1 + kept = set(group.nodes) - set(doomed) + keeper = kept.pop() + assert all(len(keeper.path) <= len(n.path) for n in doomed) + + +def test_digest_helpers(tmp_path): + a = tmp_path / "a.bin" + b = tmp_path / "b.bin" + a.write_bytes(b"hello world" * 100) + b.write_bytes(b"hello world" * 100) + + assert duplicates.full_digest(str(a)) == duplicates.full_digest(str(b)) + assert duplicates.sample_digest(str(a)) == duplicates.sample_digest(str(b)) + + b.write_bytes(b"different" * 100) + assert duplicates.full_digest(str(a)) != duplicates.full_digest(str(b)) + + +def test_digests_of_unreadable_file_are_none(tmp_path): + missing = str(tmp_path / "nope.bin") + assert duplicates.full_digest(missing) is None + assert duplicates.sample_digest(missing) is None + + +def test_large_files_differing_only_in_the_middle(tmp_path): + """The cheap sample pass must not be trusted on its own.""" + head_tail_same = b"H" * 70000 + b"M" * 10 + b"T" * 70000 + other = b"H" * 70000 + b"N" * 10 + b"T" * 70000 + (tmp_path / "one.bin").write_bytes(head_tail_same) + (tmp_path / "two.bin").write_bytes(other) + + assert duplicates.sample_digest(str(tmp_path / "one.bin")) == \ + duplicates.sample_digest(str(tmp_path / "two.bin")) + assert duplicates.find_duplicates(scan(tmp_path), min_size=1) == [] diff --git a/tests/test_gui.py b/tests/test_gui.py index 2aad1c8..cabfbc1 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -19,12 +19,6 @@ if sys.platform != "win32" and not os.environ.get("DISPLAY"): pytest.skip("no display available", allow_module_level=True) -try: - _probe = tk.Tk() - _probe.destroy() -except Exception as exc: # pragma: no cover - environment dependent - pytest.skip(f"cannot open a Tk display: {exc}", allow_module_level=True) - import treemap_render from scanner import TreeScanner @@ -66,21 +60,27 @@ def gallery(tmp_path): return root -@pytest.fixture -def gui(tmp_path, sample_tree, monkeypatch): - """A FolderLensApp with an already-scanned tree and isolated settings.""" - # keep the test off the developer's real settings file - monkeypatch.setenv("APPDATA", str(tmp_path / "cfg")) - monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "cfg")) +@pytest.fixture(scope="session") +def app_window(tmp_path_factory): + """One application window for the whole session. + + Every test used to build its own root window. Creating and tearing down + that many Tcl interpreters in a single process is flaky on CI runners — + it intermittently fails with "Can't find a usable init.tcl" partway + through the suite — so the window is made once and reset between tests. + """ + cfg = tmp_path_factory.mktemp("folderlens-cfg") + os.environ["APPDATA"] = str(cfg) # keep off the real settings file + os.environ["XDG_CONFIG_HOME"] = str(cfg) import app as appmod - root = scan_sync(sample_tree) - assert root is not None + try: + win = appmod.FolderLensApp(initial_path=None) # no scan kicked off + except tk.TclError as exc: # pragma: no cover - environment + pytest.skip(f"cannot open a Tk display: {exc}") - win = appmod.FolderLensApp(initial_path=None) # no scan kicked off win.geometry("1000x700+0+0") - win.root_node = root try: yield win finally: @@ -90,6 +90,36 @@ def gui(tmp_path, sample_tree, monkeypatch): pass +@pytest.fixture +def gui(app_window, sample_tree): + """The shared window, reset and pointed at a freshly scanned tree.""" + win = app_window + + root = scan_sync(sample_tree) + assert root is not None + win.root_node = root + + # clear anything a previous test left behind + win.search_var.set("") + win.search_query = "" + win.treemap_stack = [] + win.dup_groups = [] + win._hover_tile = None + win._treemap_image = None + + # tests may unbind the resize handler to drive the reflow directly + win.unbind("") + win.bind("", win._on_window_configure, add="+") + win._toolbar_narrow = None + win._reflow_toolbar(win.NARROW_WIDTH + 300) + + show(win, "Tree") + try: + yield win + finally: + win._clear_body() + + def show(win, view): win.active_view = view win.view_switch.set(view) @@ -420,3 +450,63 @@ def test_viewer_annotation_actions_stay_visible_when_narrow(gui, gallery): assert packed_in(viewer.tools_left) == str(viewer.tools_row_a) finally: viewer.destroy() + + +# ------------------------------------------------------------- duplicates view + +def test_duplicates_view_renders_and_takes_selection(gui, tmp_path): + """Duplicates is a selection view, so Zip/Delete must reach it.""" + import duplicates as dup + + show(gui, "Duplicates") + assert gui.dup_tree is not None + + # inject a finished result rather than hashing during the test + files = [n for n in gui.root_node.children if not n.is_dir] + assert len(files) >= 2 + gui.dup_groups = [dup.DuplicateGroup(size=files[0].size, nodes=files[:2])] + gui._fill_duplicates() + gui.update_idletasks() + + groups = gui.dup_tree.get_children() + assert len(groups) == 1 + rows = gui.dup_tree.get_children(groups[0]) + assert len(rows) == 2 + + gui.dup_tree.selection_set(rows[0]) + selection = gui._top_level_selection() + assert len(selection) == 1 + assert selection[0][1] in files + + +def test_duplicates_is_offered_as_a_view(gui): + assert "Duplicates" in gui.VIEWS + + +def test_treemap_keeps_an_exportable_image(gui): + show(gui, "Treemap") + gui.treemap_canvas.configure(width=520, height=380) + gui.update_idletasks() + gui._draw_treemap() + assert gui._treemap_image is not None + assert gui._treemap_image.size == (gui.treemap_canvas.winfo_width(), + gui.treemap_canvas.winfo_height()) + + +def test_hover_does_not_rerender_the_treemap(gui): + """Regression: the highlight used to be baked into the image, so every + mouse move across a tile boundary re-composited the whole map.""" + show(gui, "Treemap") + gui.treemap_canvas.configure(width=640, height=440) + gui.update_idletasks() + gui._draw_treemap() + + before = gui._treemap_photo + assert gui._tiles + for tile in gui._tiles[:10]: + event = type("E", (), {"x": int(tile.x + tile.w / 2), + "y": int(tile.y + tile.h / 2)})() + gui._treemap_hover(event) + + assert gui._treemap_photo is before, "the treemap image was rebuilt on hover" + assert gui._highlight_id is not None, "no highlight outline was drawn" diff --git a/tests/test_scanner.py b/tests/test_scanner.py index 6adaba0..38abb9c 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -6,7 +6,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from scanner import TreeScanner, FolderScanner, QuickScanner +from scanner import TreeScanner, FolderScanner, QuickScanner, Node @pytest.fixture @@ -153,3 +153,25 @@ def test_quick_scanner(sample_tree): dirs = [i for i in items if i.is_directory] assert len(dirs) == 1 assert dirs[0].size == 0 + + +def test_node_uses_slots_to_keep_the_tree_small(): + """One Node exists per file on disk, so the per-instance __dict__ matters.""" + node = Node(path="/x/a.txt", name="a.txt", is_dir=False, size=1) + assert not hasattr(node, "__dict__"), "Node grew a __dict__ again" + assert hasattr(Node, "__slots__") + + +def test_files_share_one_empty_children_container(): + """Files can never have children; giving each its own list wasted memory.""" + a = Node(path="/x/a.txt", name="a.txt", is_dir=False) + b = Node(path="/x/b.txt", name="b.txt", is_dir=False) + assert a.children is b.children + assert len(a.children) == 0 + + # directories still get their own mutable list + d1 = Node(path="/x/d1", name="d1", is_dir=True) + d2 = Node(path="/x/d2", name="d2", is_dir=True) + assert d1.children is not d2.children + d1.children.append(a) + assert d2.children == [] diff --git a/tests/test_trash.py b/tests/test_trash.py new file mode 100644 index 0000000..d7120f7 --- /dev/null +++ b/tests/test_trash.py @@ -0,0 +1,63 @@ +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import trash + + +def test_missing_file_is_reported_not_raised(tmp_path): + moved, message = trash.send_to_trash(str(tmp_path / "nope.txt")) + assert moved is False + assert "not found" in message.lower() + + +def test_unique_name_avoids_collisions(tmp_path): + (tmp_path / "a.txt").write_text("x") + (tmp_path / "a.1.txt").write_text("x") + name = trash._unique_name(str(tmp_path), "a.txt") + assert name not in ("a.txt", "a.1.txt") + assert not os.path.exists(tmp_path / name) + + +def test_xdg_trash_moves_the_file(tmp_path, monkeypatch): + if sys.platform == "win32": + return # the shell API is exercised on Windows, not here + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + (tmp_path / "data").mkdir() + + victim = tmp_path / "victim.txt" + victim.write_text("delete me") + + moved, message = trash.send_to_trash(str(victim)) + assert moved is True, message + assert not victim.exists(), "file was not moved out of the way" + + trashed = tmp_path / "data" / "Trash" / "files" / "victim.txt" + assert trashed.exists(), "file did not land in the trash" + assert trashed.read_text() == "delete me" + + info = tmp_path / "data" / "Trash" / "info" / "victim.txt.trashinfo" + assert info.exists(), "no restore metadata written" + assert "Path=" in info.read_text() + + +def test_second_file_with_the_same_name_does_not_overwrite(tmp_path, monkeypatch): + if sys.platform == "win32": + return + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "data")) + (tmp_path / "data").mkdir() + + for content in ("first", "second"): + folder = tmp_path / content + folder.mkdir() + target = folder / "same.txt" + target.write_text(content) + assert trash.send_to_trash(str(target))[0] + + files = os.listdir(tmp_path / "data" / "Trash" / "files") + assert len(files) == 2, f"a file was overwritten in the trash: {files}" + + +def test_is_supported_reports_a_bool(): + assert isinstance(trash.is_supported(), bool) diff --git a/tests/test_treemap_render.py b/tests/test_treemap_render.py index d0b8cf4..f555f39 100644 --- a/tests/test_treemap_render.py +++ b/tests/test_treemap_render.py @@ -183,3 +183,78 @@ def test_header_skipped_on_tiles_too_small_to_show_one(): min_area=1, max_depth=6, header=18.0) for tile in tiles: assert tile.h >= 0 + + +# ------------------------------------------------------------- optimizations + +def test_tiny_tiles_are_filled_flat(): + """Sub-8px tiles get a flat fill: the gradient is invisible at that size + and shading thousands of them dominated the render.""" + import treemap_render as tr + assert tr.FLAT_FILL_BELOW >= 4 + img = tr.render_treemap(tiles_for(60, 40), 60, 40) + assert img.size == (60, 40) + + +def test_cushion_sources_are_shared_per_colour_not_per_tile(): + """Cushions come from a small per-colour pyramid, so the cache must grow + with the number of colours and levels, never with the number of tiles.""" + import treemap_render as tr + tr._cushion_rgb_cache.clear() + + # many same-type files: lots of tiles, exactly one colour between them + root = Node(path="/root", name="root", is_dir=True) + root.children = [ + Node(path=f"/root/clip{i}.mp4", name=f"clip{i}.mp4", is_dir=False, + size=1000 + i, parent=root) + for i in range(120) + ] + root.size = sum(c.size for c in root.children) + + tiles = analysis.build_treemap(root, 0, 0, 1200, 900, min_area=1, max_depth=4) + tr.render_treemap(tiles, 1200, 900, tr.RenderOptions(show_thumbnails=False)) + + shaded = [t for t in tiles + if int(t.w) >= tr.FLAT_FILL_BELOW and int(t.h) >= tr.FLAT_FILL_BELOW] + assert len(shaded) > 20, "expected plenty of shaded tiles in this layout" + # one colour, so at most one entry per pyramid level regardless of tiles + assert len(tr._cushion_rgb_cache) <= len(tr._CUSHION_LEVELS) + assert len(tr._cushion_rgb_cache) < len(shaded) + + +def test_cushion_level_picks_the_nearest_size_up(): + import treemap_render as tr + assert tr._cushion_level(10) == 16 + assert tr._cushion_level(16) == 16 + assert tr._cushion_level(17) == 32 + assert tr._cushion_level(10_000) == tr._CUSHION_LEVELS[-1] + + +def test_shade_luts_are_bytes(): + """A list LUT makes Pillow re-round 256 entries on every single call.""" + import treemap_render as tr + for lut in tr._shade_luts((120, 30, 200)): + assert isinstance(lut, bytes) + assert len(lut) == 256 + + +def test_tile_colour_is_cached_by_extension(): + import treemap_render as tr + from scanner import Node + tr._color_by_ext.clear() + a = Node(path="/x/one.mp4", name="one.mp4", is_dir=False, size=1) + b = Node(path="/y/two.mp4", name="two.mp4", is_dir=False, size=1) + assert tr.tile_color(a, True) == tr.tile_color(b, True) + assert len(tr._color_by_ext) == 1 + + +def test_thumbnail_sizes_are_bucketed_so_resizing_reuses_decodes(): + """Requesting the exact tile size meant every window resize re-decoded + every image; buckets make a resize reuse what is already cached.""" + from thumbnails import bucket_size, SIZE_BUCKETS + assert bucket_size((20, 15)) == bucket_size((30, 28)) + assert bucket_size((10, 10))[0] == SIZE_BUCKETS[0] + assert bucket_size((5000, 5000))[0] == SIZE_BUCKETS[-1] + # buckets never shrink the request below what was asked for + for size in [(9, 9), (33, 20), (200, 130)]: + assert bucket_size(size)[0] >= max(size) diff --git a/thumbnails.py b/thumbnails.py index a680508..7569288 100644 --- a/thumbnails.py +++ b/thumbnails.py @@ -22,10 +22,33 @@ CacheKey = Tuple[str, int, int, float] +def bucket_size(size: Tuple[int, int]) -> Tuple[int, int]: + """Round a requested size up to the next cache bucket. + + Tiles are all slightly different sizes and every window resize changes + them again. Keying the cache on the exact request meant re-decoding every + image on every resize; snapping to a handful of buckets means a resize + almost always reuses what has already been decoded, and the extra pixels + are thrown away by the downscale that follows. + """ + want = max(size[0], size[1], 1) + for bucket in SIZE_BUCKETS: + if want <= bucket: + return (bucket, bucket) + return (SIZE_BUCKETS[-1], SIZE_BUCKETS[-1]) + + +SIZE_BUCKETS = (32, 64, 128, 256, 512) + + class ThumbnailCache: - def __init__(self, max_items: int = 600, workers: int = 3): - self.max_items = max_items + """Decoded thumbnails, bounded by total pixels rather than entry count: + a handful of 512px thumbnails cost far more than many 32px ones.""" + + def __init__(self, pixel_budget: int = 24_000_000, workers: int = 3): + self.pixel_budget = pixel_budget self._cache: "OrderedDict[CacheKey, Optional[Image.Image]]" = OrderedDict() + self._pixels = 0 self._pending: set = set() self._lock = threading.Lock() self._executor = ThreadPoolExecutor(max_workers=workers, @@ -50,7 +73,7 @@ def _mtime(path: str) -> float: def get(self, path: str, size: Tuple[int, int]) -> Optional[Image.Image]: """Return a cached thumbnail, or None. Never blocks, never decodes.""" - key = self._key(path, size, self._mtime(path)) + key = self._key(path, bucket_size(size), self._mtime(path)) with self._lock: if key in self._cache: self._cache.move_to_end(key) @@ -63,18 +86,19 @@ def request(self, path: str, size: Tuple[int, int]) -> Optional[Image.Image]: if self._closed or not is_image_file(path): return None - cached = self.get(path, size) + bucket = bucket_size(size) + cached = self.get(path, bucket) if cached is not None: return cached - key = self._key(path, size, self._mtime(path)) + key = self._key(path, bucket, self._mtime(path)) with self._lock: if key in self._cache or key in self._pending: return None self._pending.add(key) try: - self._executor.submit(self._load, key, path, size) + self._executor.submit(self._load, key, path, bucket) except RuntimeError: # executor already shut down with self._lock: self._pending.discard(key) @@ -84,6 +108,13 @@ def _load(self, key: CacheKey, path: str, size: Tuple[int, int]): image = None try: with Image.open(path) as src: + # draft() lets the JPEG decoder skip most of the work when we + # only need a small thumbnail: decode at 1/2, 1/4 or 1/8 scale + # instead of full resolution and then shrinking. + try: + src.draft("RGB", size) + except Exception: + pass src = ImageOps.exif_transpose(src) src.thumbnail(size, Image.Resampling.LANCZOS) image = src.convert("RGB") @@ -92,10 +123,7 @@ def _load(self, key: CacheKey, path: str, size: Tuple[int, int]): finally: with self._lock: self._pending.discard(key) - self._cache[key] = image - self._cache.move_to_end(key) - while len(self._cache) > self.max_items: - self._cache.popitem(last=False) + self._store(key, image) if image is not None and self._on_ready and not self._closed: try: @@ -103,9 +131,28 @@ def _load(self, key: CacheKey, path: str, size: Tuple[int, int]): except Exception: pass + def _store(self, key: CacheKey, image: Optional[Image.Image]): + """Insert and evict down to the pixel budget. Caller holds the lock.""" + old = self._cache.pop(key, None) + if old is not None: + self._pixels -= old.width * old.height + self._cache[key] = image + if image is not None: + self._pixels += image.width * image.height + + while self._pixels > self.pixel_budget and len(self._cache) > 1: + _, evicted = self._cache.popitem(last=False) + if evicted is not None: + self._pixels -= evicted.width * evicted.height + + @property + def pixels(self) -> int: + return self._pixels + def clear(self): with self._lock: self._cache.clear() + self._pixels = 0 def close(self): self._closed = True diff --git a/trash.py b/trash.py new file mode 100644 index 0000000..cfd937b --- /dev/null +++ b/trash.py @@ -0,0 +1,126 @@ +"""Move files to the Recycle Bin instead of deleting them outright. + +A disk cleaner that can only delete permanently is a sharp tool to hand +someone who is skimming a list of files. On Windows this uses the shell's own +undo-able delete; elsewhere it follows the XDG trash spec well enough for the +desktop to show the files in its bin. + +`send_to_trash` never raises: it reports whether the file made it to the bin, +and the caller decides whether to fall back to a permanent delete. +""" +import os +import shutil +import sys +import time +from typing import Tuple +from urllib.parse import quote + + +def is_supported() -> bool: + if sys.platform == "win32": + return True + return bool(_xdg_trash_dir()) + + +def send_to_trash(path: str) -> Tuple[bool, str]: + """Try to move `path` to the recycle bin. + + Returns (moved, message). A False result is not an error the caller must + surface: it means the bin was unavailable and a permanent delete is the + only remaining option. + """ + if not os.path.exists(path): + return False, "File not found" + + if sys.platform == "win32": + return _windows_recycle(os.path.abspath(path)) + return _xdg_trash(os.path.abspath(path)) + + +# --------------------------------------------------------------------- windows + +def _windows_recycle(path: str) -> Tuple[bool, str]: + import ctypes + from ctypes import wintypes + + FO_DELETE = 0x0003 + FOF_ALLOWUNDO = 0x0040 + FOF_NOCONFIRMATION = 0x0010 + FOF_NOERRORUI = 0x0400 + FOF_SILENT = 0x0004 + + class SHFILEOPSTRUCTW(ctypes.Structure): + _fields_ = [ + ("hwnd", wintypes.HWND), + ("wFunc", wintypes.UINT), + ("pFrom", wintypes.LPCWSTR), + ("pTo", wintypes.LPCWSTR), + ("fFlags", ctypes.c_uint16), + ("fAnyOperationsAborted", wintypes.BOOL), + ("hNameMappings", ctypes.c_void_p), + ("lpszProgressTitle", wintypes.LPCWSTR), + ] + + # the path list is double-NUL terminated + op = SHFILEOPSTRUCTW( + hwnd=None, + wFunc=FO_DELETE, + pFrom=path + "\0\0", + pTo=None, + fFlags=FOF_ALLOWUNDO | FOF_NOCONFIRMATION | FOF_NOERRORUI | FOF_SILENT, + fAnyOperationsAborted=False, + hNameMappings=None, + lpszProgressTitle=None, + ) + try: + result = ctypes.windll.shell32.SHFileOperationW(ctypes.byref(op)) + except Exception as exc: # pragma: no cover - windows only + return False, f"Recycle Bin unavailable: {exc}" + + if result != 0: + return False, f"Recycle Bin refused the file (code {result})" + if op.fAnyOperationsAborted: + return False, "Cancelled" + return True, "Moved to Recycle Bin" + + +# ------------------------------------------------------------------------ xdg + +def _xdg_trash_dir() -> str: + home = os.path.expanduser("~") + base = os.environ.get("XDG_DATA_HOME") or os.path.join(home, ".local", "share") + trash = os.path.join(base, "Trash") + return trash if os.path.isdir(os.path.dirname(trash)) else "" + + +def _unique_name(folder: str, name: str) -> str: + candidate = name + stem, ext = os.path.splitext(name) + counter = 1 + while os.path.exists(os.path.join(folder, candidate)): + candidate = f"{stem}.{counter}{ext}" + counter += 1 + return candidate + + +def _xdg_trash(path: str) -> Tuple[bool, str]: + trash = _xdg_trash_dir() + if not trash: + return False, "No trash directory available" + + files_dir = os.path.join(trash, "files") + info_dir = os.path.join(trash, "info") + try: + os.makedirs(files_dir, exist_ok=True) + os.makedirs(info_dir, exist_ok=True) + + name = _unique_name(files_dir, os.path.basename(path)) + with open(os.path.join(info_dir, name + ".trashinfo"), "w", encoding="utf-8") as f: + f.write("[Trash Info]\n") + f.write(f"Path={quote(path)}\n") + f.write(f"DeletionDate={time.strftime('%Y-%m-%dT%H:%M:%S')}\n") + + shutil.move(path, os.path.join(files_dir, name)) + return True, "Moved to Trash" + except OSError as exc: + return False, f"Trash unavailable: {exc}" diff --git a/treemap_render.py b/treemap_render.py index dfc0802..0464562 100644 --- a/treemap_render.py +++ b/treemap_render.py @@ -14,22 +14,34 @@ Compositing into one image is also far cheaper than thousands of live canvas items, which is what made the old view stutter on big trees. """ +import os +from collections import OrderedDict from dataclasses import dataclass from typing import Callable, Dict, Optional, Sequence, Tuple from PIL import Image, ImageDraw, ImageFont, ImageOps from file_utils import get_file_category, is_image_file -from thumbnails import cover_box WHITE = (255, 255, 255) BLACK = (0, 0, 0) # 256x256 'L' bump, bright in the middle: the cushion highlight. _CUSHION_BASE = ImageOps.invert(Image.radial_gradient("L")) -_cushion_cache: Dict[Tuple[int, int], Image.Image] = {} +_cushion_cache: "OrderedDict[Tuple[int, int], Image.Image]" = OrderedDict() +_cushion_pixels = 0 +_CUSHION_PIXEL_BUDGET = 8_000_000 # ~8 MB of masks, then evict oldest +_lut_cache: Dict[Tuple[int, int, int], Tuple[list, list, list]] = {} _font_cache: Dict[int, object] = {} +# Below this a tile is a few pixels across: the cushion gradient cannot be +# seen, so it is filled flat. Saves the bulk of the work on dense trees +# without any visible difference. +FLAT_FILL_BELOW = 8 + +SHADE_MIX = 0.42 # how dark the tile edges go +LIGHT_MIX = 0.30 # how bright the centre highlight goes + @dataclass class RenderOptions: @@ -44,16 +56,92 @@ class RenderOptions: def _cushion(w: int, h: int) -> Image.Image: + """Cushion mask at a given size, cached under a pixel budget. + + Keyed by exact size, so a tree with thousands of distinct tile sizes used + to be able to grow this without limit; it is now an LRU bounded by total + pixels rather than entry count, since a few large masks cost far more than + many small ones. + """ + global _cushion_pixels key = (w, h) mask = _cushion_cache.get(key) - if mask is None: - mask = _CUSHION_BASE.resize((w, h), Image.Resampling.BILINEAR) - if len(_cushion_cache) > 4000: - _cushion_cache.clear() - _cushion_cache[key] = mask + if mask is not None: + _cushion_cache.move_to_end(key) + return mask + + mask = _CUSHION_BASE.resize((w, h), Image.Resampling.BILINEAR) + _cushion_cache[key] = mask + _cushion_pixels += w * h + while _cushion_pixels > _CUSHION_PIXEL_BUDGET and len(_cushion_cache) > 1: + (ow, oh), _ = _cushion_cache.popitem(last=False) + _cushion_pixels -= ow * oh return mask +def _shade_luts(color: Tuple[int, int, int]): + """Per-channel lookup tables mapping cushion brightness to tile colour. + + Applying three LUTs to the mask replaces allocating four temporary images + and running two blends plus a composite for every single tile. + """ + luts = _lut_cache.get(color) + if luts is None: + low = [c * (1.0 - SHADE_MIX) for c in color] + high = [c + (255 - c) * LIGHT_MIX for c in color] + # bytes, not list: Pillow re-rounds a list LUT in Python on every + # call, which dominated the render on trees with thousands of tiles + luts = tuple( + bytes(min(255, max(0, int(low[i] + (high[i] - low[i]) * v / 255.0))) + for v in range(256)) + for i in range(3) + ) + if len(_lut_cache) > 512: + _lut_cache.clear() + _lut_cache[color] = luts + return luts + + +# Cushions are kept per colour at a few resolutions. Scaling every tile down +# from one large source meant reading the whole source for even a 20px tile; +# picking the nearest level keeps each resize close to 1:1. +_CUSHION_LEVELS = (16, 32, 64, 128, 256) +_cushion_rgb_cache: Dict[Tuple[Tuple[int, int, int], int], Image.Image] = {} + + +def _cushion_level(size: int) -> int: + for level in _CUSHION_LEVELS: + if size <= level: + return level + return _CUSHION_LEVELS[-1] + + +def _cushion_rgb(color: Tuple[int, int, int], level: int) -> Image.Image: + """A finished cushion tile for one colour at one pyramid level. + + Only a dozen or so colours exist (one per file category), so these are + built a handful of times per session and every tile is then one cheap + resize. Shading each tile from scratch cost several passes over its pixels. + """ + key = (color, level) + tile = _cushion_rgb_cache.get(key) + if tile is None: + mask = _CUSHION_BASE.resize((level, level), Image.Resampling.BILINEAR) + r, g, b = _shade_luts(color) + tile = Image.merge("RGB", (mask.point(r), mask.point(g), mask.point(b))) + if len(_cushion_rgb_cache) > 128: + _cushion_rgb_cache.clear() + _cushion_rgb_cache[key] = tile + return tile + + +def _cushion_tile(color: Tuple[int, int, int], w: int, h: int) -> Image.Image: + source = _cushion_rgb(color, _cushion_level(max(w, h))) + if source.size == (w, h): + return source.copy() + return source.resize((w, h), Image.Resampling.BILINEAR) + + def _font(px: int): px = max(7, min(px, 40)) font = _font_cache.get(px) @@ -77,10 +165,22 @@ def _rgb(color: str) -> Tuple[int, int, int]: return tuple(int(color[i:i + 2], 16) for i in (0, 2, 4)) +_color_by_ext: Dict[str, Tuple[int, int, int]] = {} + + def tile_color(node, dark_mode: bool) -> Tuple[int, int, int]: if node.is_dir: return (63, 63, 70) if dark_mode else (203, 213, 225) - return _rgb(get_file_category(node.path)['color']) + # a tile's colour depends only on its extension, and a big tree asks the + # same question thousands of times + ext = os.path.splitext(node.path)[1].lower() + color = _color_by_ext.get(ext) + if color is None: + color = _rgb(get_file_category(node.path)['color']) + if len(_color_by_ext) > 4000: + _color_by_ext.clear() + _color_by_ext[ext] = color + return color def _draw_label(draw: ImageDraw.ImageDraw, x: float, y: float, text: str, @@ -112,44 +212,42 @@ def render_treemap(tiles: Sequence, width: int, height: int, continue x, y = int(tile.x), int(tile.y) node = tile.node + color = tile_color(node, opts.dark_mode) + + # too small for the gradient to be visible: fill flat, no allocation + if tw < FLAT_FILL_BELOW or th < FLAT_FILL_BELOW: + canvas.paste(color, (x, y, x + tw, y + th)) + continue patch = None if (opts.show_thumbnails and thumb_provider is not None and not node.is_dir and is_image_file(node.path) and tw >= opts.min_thumb and th >= opts.min_thumb): - thumb = thumb_provider(node.path, (max(tw, 64), max(th, 64))) + thumb = thumb_provider(node.path, (tw, th)) if thumb is not None: - # fill the tile edge-to-edge, then centre-crop the overflow - cw, ch = cover_box(thumb.width, thumb.height, tw, th) - scaled = thumb.resize((cw, ch), Image.Resampling.BILINEAR) - left = max(0, (cw - tw) // 2) - top = max(0, (ch - th) // 2) - patch = scaled.crop((left, top, left + tw, top + th)) + # fill the tile edge-to-edge, cropping the overflow + patch = ImageOps.fit(thumb, (tw, th), method=Image.Resampling.BILINEAR) if patch is None: - base = Image.new("RGB", (tw, th), tile_color(node, opts.dark_mode)) - mask = _cushion(tw, th) - lit = Image.blend(base, Image.new("RGB", (tw, th), WHITE), 0.30) - shade = Image.blend(base, Image.new("RGB", (tw, th), BLACK), 0.42) - patch = Image.composite(lit, shade, mask) + patch = _cushion_tile(color, tw, th) else: # keep a hint of the cushion so thumbnails still read as tiles, # but stay light enough that the picture is what you notice - mask = _cushion(tw, th) shade = Image.blend(patch, Image.new("RGB", (tw, th), BLACK), 0.28) - patch = Image.composite(patch, shade, mask) + patch = Image.composite(patch, shade, _cushion(tw, th)) canvas.paste(patch, (x, y)) draw = ImageDraw.Draw(canvas, "RGBA") - # thin separators + # thin separators; skip tiles too small for an outline to read + separator = border + (140,) for tile in tiles: tw, th = int(tile.w), int(tile.h) - if tw < 3 or th < 3: + if tw < 5 or th < 5: continue draw.rectangle([int(tile.x), int(tile.y), int(tile.x) + tw - 1, int(tile.y) + th - 1], - outline=border + (140,), width=1) + outline=separator, width=1) if opts.show_labels: # files first... diff --git a/version.py b/version.py index a20b28d..c2531c7 100644 --- a/version.py +++ b/version.py @@ -1,4 +1,4 @@ -VERSION = "3.1.0" +VERSION = "3.2.0" GITHUB_OWNER = "MrHakan" GITHUB_REPO = "FolderLens"