From dca1e51fcce7311db9eb67143aca6af5f33038f8 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 4 Aug 2026 14:55:36 -0700 Subject: [PATCH] Add shallow-clone (depth) support to GitFetcher Introduce a `depth` source option on git sources so components can be fetched with a shallow clone instead of downloading the full history. This can significantly cut fetch time and disk usage for large repositories where full history is not needed. When `source git: url, depth: N` (positive integer) is set, the fetcher initializes an empty repo and fetches the *resolved revision* (SHA) directly at that depth, rather than `git clone`ing the full history: git init --quiet . git fetch --depth N Fetching the resolved SHA (instead of a branch/tag ref) means shallow mode works for tag, branch, and SHA pins alike, and is immune to a pinned branch tip moving between resolution and fetch. It does require the remote to allow fetching a reachable SHA (uploadpack.allowReachableSHA1InWant / allowAnySHA1InWant). For submodules, shallow mode uses `submodule update --init` (since there was no `clone --recursive`) but deliberately does not pass `--depth`: a shallow submodule fetch targets the submodule's branch tip and fails when the pinned submodule commit is not near it, so submodules are fetched at full depth. A missing or non-positive depth preserves the existing full-clone behavior, so this is fully backward compatible. Components whose build depends on git history (e.g. `git describe`) should not use this option. Co-Authored-By: Claude Opus 4.8 Signed-off-by: Stan Hu --- lib/omnibus/fetchers/git_fetcher.rb | 52 +++++++++- lib/omnibus/software.rb | 10 +- spec/unit/fetchers/git_fetcher_spec.rb | 125 +++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 5 deletions(-) diff --git a/lib/omnibus/fetchers/git_fetcher.rb b/lib/omnibus/fetchers/git_fetcher.rb index bfce86728..37c83ec5d 100644 --- a/lib/omnibus/fetchers/git_fetcher.rb +++ b/lib/omnibus/fetchers/git_fetcher.rb @@ -112,6 +112,23 @@ def clone_submodules? source[:submodules] || false end + # + # The depth to use for a shallow fetch, or nil for a full clone. A shallow + # fetch is requested by setting a positive `depth` source option. + # + # In shallow mode the resolved revision (SHA) is fetched directly at this + # depth, so it works for tag, branch, and SHA pins alike. The build must + # not depend on git history (e.g. `git describe`), and the remote must + # allow fetching a reachable SHA + # (uploadpack.allowReachableSHA1InWant / allowAnySHA1InWant). + # + # @return [Integer, nil] + # + def clone_depth + depth = source[:depth] + depth if depth.to_i > 0 + end + # # Determine if a directory is empty # @@ -145,8 +162,19 @@ def cloned? # @return [void] # def git_clone - retry_block("git clone", [CommandTimeout, CommandFailed]) do - git("clone#{" --recursive" if clone_submodules?} #{source_url} .") + if clone_depth + # A shallow fetch needs an exact revision, so initialize an empty repo + # and fetch the resolved SHA directly rather than `git clone`ing the + # full history. This works for tag, branch, and SHA pins alike and is + # immune to a pinned branch tip moving between resolution and fetch. + retry_block("git init", [CommandTimeout, CommandFailed]) do + git("init --quiet .") + end + git_fetch + else + retry_block("git clone", [CommandTimeout, CommandFailed]) do + git("clone#{" --recursive" if clone_submodules?} #{source_url} .") + end end end @@ -161,8 +189,18 @@ def git_checkout # support the --detach flag. git("checkout #{resolved_version} -f -q") if clone_submodules? + # In shallow mode there was no `clone --recursive`, so submodules are + # neither initialized nor populated yet; --init handles that. We do NOT + # pass --depth: `git submodule update --depth` shallow-fetches the + # submodule's branch and fails when the pinned submodule commit is not + # near that branch's tip (the same non-tip problem this fetcher avoids + # for the top-level repo by fetching the resolved SHA). Submodules are + # therefore fetched at full depth. No depth-pinned component currently + # uses submodules. + submodule_cmd = "submodule update --recursive" + submodule_cmd = "submodule update --init --recursive" if clone_depth retry_block("git submodule update", [CommandTimeout, CommandFailed]) do - git("submodule update --recursive") + git(submodule_cmd) end end end @@ -173,8 +211,14 @@ def git_checkout # @return [void] # def git_fetch - fetch_cmd = "fetch #{source_url} #{described_version}" + # In shallow mode fetch the resolved SHA directly: it is immutable, so + # this is race-free and works whether the pin is a tag, branch, or SHA. + # Otherwise fetch the pinned ref and let git_checkout resolve it. + ref = clone_depth ? resolved_version : described_version + fetch_cmd = +"fetch" + fetch_cmd << " --depth #{clone_depth}" if clone_depth fetch_cmd << " --recurse-submodules=on-demand" if clone_submodules? + fetch_cmd << " #{source_url} #{ref}" retry_block("git fetch", [CommandTimeout, CommandFailed]) do git(fetch_cmd) end diff --git a/lib/omnibus/software.rb b/lib/omnibus/software.rb index 1de6a7db2..45256bf22 100644 --- a/lib/omnibus/software.rb +++ b/lib/omnibus/software.rb @@ -320,6 +320,13 @@ def dependency(val) # @option val [Boolean] :submodules (false) # clone git submodules # + # @option val [Integer] :depth (nil) + # when set to a positive integer, perform a shallow fetch of the resolved + # revision at this depth instead of cloning the full history. Works for + # tag, branch, and SHA pins. The component's build must not depend on git + # history (e.g. `git describe`), and the remote must allow fetching a + # reachable SHA (uploadpack.allowReachableSHA1InWant / allowAnySHA1InWant). + # # If multiple checksum types are provided, only the strongest will be used. # # @return [Hash] @@ -338,7 +345,8 @@ def source(val = NULL) :md5, :sha1, :sha256, :sha512, # hash type - common to all fetchers :cookie, :warning, :unsafe, :extract, :cached_name, :authorization, :internal, # used by net_fetcher :options, # used by path_fetcher - :submodules # used by git_fetcher + :submodules, # used by git_fetcher + :depth # used by git_fetcher ] unless extra_keys.empty? raise InvalidValue.new(:source, diff --git a/spec/unit/fetchers/git_fetcher_spec.rb b/spec/unit/fetchers/git_fetcher_spec.rb index 5f41d55c1..f78c0925f 100644 --- a/spec/unit/fetchers/git_fetcher_spec.rb +++ b/spec/unit/fetchers/git_fetcher_spec.rb @@ -123,5 +123,130 @@ module Omnibus expect(subject.version_for_cache).to eq("revision:123abcd1234") end end + + describe "#git_clone" do + let(:git_source) { "https://example.com/repo.git" } + let(:locked_source) { { git: git_source } } + let(:manifest_entry) do + double(ManifestEntry, + name: "software", + locked_version: "123abcd1234", + described_version: "v1.2.3", + locked_source: locked_source) + end + + context "when depth is not set" do + it "performs a full clone" do + expect(subject).to receive(:git).with("clone #{git_source} .") + subject.send(:git_clone) + end + + context "with submodules" do + let(:locked_source) { { git: git_source, submodules: true } } + + it "clones recursively" do + expect(subject).to receive(:git).with("clone --recursive #{git_source} .") + subject.send(:git_clone) + end + end + end + + context "when depth is not positive" do + let(:locked_source) { { git: git_source, depth: 0 } } + + it "performs a full clone" do + expect(subject).to receive(:git).with("clone #{git_source} .") + subject.send(:git_clone) + end + end + + context "when depth is set" do + let(:locked_source) { { git: git_source, depth: 1 } } + + it "initializes an empty repo and fetches the resolved revision" do + expect(subject).to receive(:git).with("init --quiet .") + expect(subject).to receive(:git_fetch) + subject.send(:git_clone) + end + end + end + + describe "#git_checkout" do + let(:git_source) { "https://example.com/repo.git" } + let(:locked_source) { { git: git_source, submodules: true } } + let(:manifest_entry) do + double(ManifestEntry, + name: "software", + locked_version: "123abcd1234", + described_version: "v1.2.3", + locked_source: locked_source) + end + + before { allow(subject).to receive(:git_fetch) } + + context "when depth is not set" do + it "updates submodules at full depth" do + expect(subject).to receive(:git).with("checkout 123abcd1234 -f -q") + expect(subject).to receive(:git).with("submodule update --recursive") + subject.send(:git_checkout) + end + end + + context "when depth is set" do + let(:locked_source) { { git: git_source, submodules: true, depth: 1 } } + + it "initializes submodules (at full depth to avoid non-tip failures)" do + expect(subject).to receive(:git).with("checkout 123abcd1234 -f -q") + expect(subject).to receive(:git).with("submodule update --init --recursive") + subject.send(:git_checkout) + end + end + end + + describe "#git_fetch" do + let(:git_source) { "https://example.com/repo.git" } + let(:locked_source) { { git: git_source } } + let(:manifest_entry) do + double(ManifestEntry, + name: "software", + locked_version: "123abcd1234", + described_version: "v1.2.3", + locked_source: locked_source) + end + + context "when depth is not set" do + it "fetches the described_version with full history" do + expect(subject).to receive(:git).with("fetch #{git_source} v1.2.3") + subject.send(:git_fetch) + end + end + + context "when depth is set" do + let(:locked_source) { { git: git_source, depth: 1 } } + + it "fetches the resolved revision at the requested depth" do + expect(subject).to receive(:git).with("fetch --depth 1 #{git_source} 123abcd1234") + subject.send(:git_fetch) + end + + context "with a larger depth" do + let(:locked_source) { { git: git_source, depth: 5 } } + + it "uses the requested depth" do + expect(subject).to receive(:git).with("fetch --depth 5 #{git_source} 123abcd1234") + subject.send(:git_fetch) + end + end + + context "with submodules" do + let(:locked_source) { { git: git_source, depth: 1, submodules: true } } + + it "recurses submodules" do + expect(subject).to receive(:git).with("fetch --depth 1 --recurse-submodules=on-demand #{git_source} 123abcd1234") + subject.send(:git_fetch) + end + end + end + end end end