[OP-19772] Load projects asynchronously in the project selector

app/components/header/projects/node_component.html.erb

@@ -40,7 +40,7 @@ See COPYRIGHT and LICENSE files for more details.
      end
    end %>
 
-<% if children.any? %>
+<% if children.any? || deferred? %>
   <% @component.with_sub_tree(
        label:,
        select_variant: :none,
@@ -51,15 +51,19 @@ See COPYRIGHT and LICENSE files for more details.
        data: { node_id: project.id, test_selector: "op-header-project-select--item" }
      ) do |sub| %>
     <% add_icons.call(sub) %>
-    <% children.each do |child_node| %>
-      <%= render Header::Projects::NodeComponent.new(
-            component: sub,
-            node: child_node,
-            current_project_id: @current_project_id,
-            favorited_ids: @favorited_ids,
-            jump: @jump,
-            query_terms: @query_terms
-          ) %>
+    <% if deferred? %>
+      <% sub.with_loading_spinner(src: deferred_children_path) %>
+    <% else %>
+      <% children.each do |child_node| %>
+        <%= render Header::Projects::NodeComponent.new(
+              component: sub,
+              node: child_node,
+              current_project_id: @current_project_id,
+              favorited_ids: @favorited_ids,
+              jump: @jump,
+              query_terms: @query_terms
+            ) %>
+      <% end %>
     <% end %>
   <% end %>
 <% else %>

app/components/header/projects/node_component.rb

@@ -49,6 +49,8 @@ def current? = project.id == @current_project_id
       def favorited? = @favorited_ids.include?(project.id)
       def expanded? = @node[:expanded]
       def matches_query? = @node[:matches_query]
+      def deferred_children_path = @node[:deferred_children_path]
+      def deferred? = deferred_children_path.present?
 
       def href
         @jump.present? ? helpers.project_path(project.identifier, jump: @jump) : helpers.project_path(project.identifier)

app/controllers/header/projects_controller.rb

@@ -29,17 +29,16 @@
 #++
 
 class Header::ProjectsController < ApplicationController
-  no_authorization_required! :index, :frame
+  no_authorization_required! :index, :frame, :children
 
   MAX_NUMBER_OF_PROJECTS = 300
   VALID_FILTER_MODES = %w[all favorited].freeze
 
   def index
-    @current_project_id = params[:current_project_id].presence&.to_i
-    @jump = params[:jump].presence
+    set_request_context
     @query_terms = query.split
     @projects = load_projects
-    @favorited_ids = load_favorited_ids
+    @favorited_ids = load_favorited_ids(@projects)
     @tree = build_tree(@projects)
 
     render layout: false
@@ -53,8 +52,26 @@ def frame
     ), layout: false
   end
 
+  # Renders one node's immediate children, fetched when it's expanded in the tree.
+  def children
+    set_request_context
+    parent = Project.visible.active.find(params.expect(:parent_id))
+    @path = JSON.parse(params[:path])
+
+    child_projects = Project.nearest_visible_descendants(parent, limit: MAX_NUMBER_OF_PROJECTS).to_a
+    @favorited_ids = load_favorited_ids(child_projects)
+    @children_nodes = build_tree(child_projects)
+
+    render layout: false, formats: [:html_fragment]
+  end
+
   private
 
+  def set_request_context
+    @current_project_id = params[:current_project_id].presence&.to_i
+    @jump = params[:jump].presence
+  end
+
   def query
     @query ||= params[:query].to_s.strip
   end
@@ -65,7 +82,7 @@ def filter_mode
   end
 
   def load_projects
-    projects = base_scope.to_a
+    projects = root_query_scope.to_a
     projects = ensure_current_project_present(projects)
 
     if (query.present? || filter_mode == "favorited") && projects.any?
@@ -77,6 +94,15 @@ def load_projects
     projects
   end
 
+  # Search & favorited need matches from any depth within the hierarchy.
+  # The initial tree, however, only loads the first level of hierarchy.
+  # The rest gets loaded from #children when nodes are expanded.
+  def root_query_scope
+    return base_scope if query.present? || filter_mode == "favorited"
+
+    Project.nearest_visible_descendants(limit: MAX_NUMBER_OF_PROJECTS)
+  end
+
   def base_scope
     scope = Project.visible.active.order(:lft).limit(MAX_NUMBER_OF_PROJECTS)
     query.split.each do |term|
@@ -107,8 +133,15 @@ def skip_current_project_inclusion?
     query.present? || filter_mode == "favorited" || @current_project_id.blank?
   end
 
+  # Loads each ancestor's full child set (not just the chain down to `current`), so every
+  # ancestor on the path renders exactly as if it had been expanded manually - siblings
+  # included, and with no dangling expand arrow left behind for it.
   def merge_with_ancestors(projects, current)
-    (projects + current.self_and_ancestors.visible.active.to_a).uniq(&:id).sort_by(&:lft)
+    ancestors = current.self_and_ancestors.visible.active.to_a
+    ancestor_children = (ancestors - [current]).flat_map do |ancestor|
+      Project.nearest_visible_descendants(ancestor, limit: MAX_NUMBER_OF_PROJECTS).to_a
+    end
+    (projects + ancestors + ancestor_children).uniq(&:id).sort_by(&:lft)
   end
 
   # Returns a scope for all visible, active ancestors of the given projects
@@ -130,11 +163,11 @@ def favorite_project_ids
     user_project_favorites.select(:favorited_id)
   end
 
-  def load_favorited_ids
+  def load_favorited_ids(projects)
     return Set.new unless current_user.logged?
 
     user_project_favorites
-      .where(favorited_id: @projects.map(&:id))
+      .where(favorited_id: projects.map(&:id))
       .pluck(:favorited_id)
       .to_set
   end
@@ -146,17 +179,44 @@ def user_project_favorites
   # Builds the nested tree from a flat, lft-ordered list of projects and
   # decorates each node with its query-match and expansion state.
   def build_tree(projects)
-    decorate_nodes(Project.build_projects_hierarchy(projects))
+    # Empty outside browse mode rather than skipped, so defer_children? doesn't need to know why -
+    # it just checks membership. Search/favorited already load everything eagerly. Computed once
+    # here and passed down, rather than recomputed per node.
+    project_ids_having_visible_descendants =
+      lazy_loading? ? Project.having_visible_descendants(projects).pluck(:id).to_set : Set.new
+    decorate_nodes(Project.build_projects_hierarchy(projects), project_ids_having_visible_descendants)
   end
 
-  def decorate_nodes(nodes)
+  def decorate_nodes(nodes, project_ids_having_visible_descendants)
     nodes.each do |node|
-      decorate_nodes(node[:children])
+      decorate_nodes(node[:children], project_ids_having_visible_descendants)
       node[:matches_query] = @matching_ids.nil? || @matching_ids.include?(node[:project].id)
       node[:expanded] = expanded_node?(node)
+      node[:deferred_children_path] = deferred_children_path_for(node, project_ids_having_visible_descendants)
     end
   end
 
+  def deferred_children_path_for(node, project_ids_having_visible_descendants)
+    return unless defer_children?(node, project_ids_having_visible_descendants)
+
+    children_header_projects_path(
+      parent_id: node[:project].id,
+      current_project_id: @current_project_id,
+      jump: @jump
+    )
+  end
+
+  # A node's children are deferred (shown behind a lazy-load link instead of loaded eagerly)
+  # when we're browsing (search/favorited load everything eagerly), its children aren't loaded
+  # yet, and it actually has visible descendants to show.
+  def defer_children?(node, project_ids_having_visible_descendants)
+    lazy_loading? && node[:children].empty? && project_ids_having_visible_descendants.include?(node[:project].id)
+  end
+
+  def lazy_loading?
+    query.blank? && filter_mode != "favorited"
+  end
+
   # A node is expanded so its children are revealed when:
   # - favorited mode is active (always expand to surface favorites), or
   # - a child is grafted onto this node because its real parent is hidden

app/models/projects/hierarchy.rb

@@ -93,11 +93,93 @@ def project_tree(projects, &)
       project_tree_from_hierarchy(projects_hierarchy, 0, &)
     end
 
+    # Returns `boundary`'s nearest visible/active descendants - normally its direct children,
+    # but if a child is invisible or archived, its own visible/active children are surfaced instead
+    # of hiding that whole branch (repeating as needed, so a chain of several invisible
+    # ancestors is skipped entirely).
+    #
+    # Without a boundary, this does the same thing for the whole instance: the result is the
+    # visible/active root-level projects, plus any deeper project promoted to the top level
+    # because its real ancestors are all invisible or archived.
+    def nearest_visible_descendants(boundary = nil, limit: nil)
+      # Both the root case and the within-boundary case reduce to the same question: which
+      # projects in a visible set have no ancestor also in that set? Only the set differs -
+      # unrestricted for roots, narrowed to the boundary's descendants otherwise.
+      scope = if boundary
+                boundary.descendants
+              else
+                Project.all
+              end
+
+      # With awesome_nested_set, for a list of projects ordered by "lft ASC"
+      # one part of the criteria for whether a project A is an ancestor of another
+      # project B is already given by the position in the list (A.lft < B.lft).
+      # Only projects before the current project can potentially be ancestors and need
+      # to be checked.
+      # Note that here, those that have no ancestors are of interest.
+      # We know that a project has no ancestor if its rgt is higher than all the rgt values
+      # of the preceding projects (B.rgt > A.rgt)
+      #
+      # To calculate this efficiently, a window function is used:
+      # max_preceding_rgt is the maximum rgt value of all the preceding projects
+      # (`ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING`) where the preceding projects
+      # are those with a smaller lft value (ORDER BY lft).
+      #
+      # Since the value of max_preceding_rgt is calculated on each row, it can now be compared
+      # directly within itself.
+      # The first row is a special case, as it has no preceding projects, its max_preceding_rgt is NULL, which is
+      # why the `IS NULL` condition is necessary.
+      visible_scope = scope
+                        .visible
+                        .select("
+                          *,
+                          max(rgt) OVER (ORDER BY lft ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING) AS max_preceding_rgt")
+
+      ancestor_scope = Project
+                         .with(projects: visible_scope)
+                         .where("projects.max_preceding_rgt IS NULL OR projects.rgt > projects.max_preceding_rgt")
+                         .order("projects.lft")
+
+      if limit
+        ancestor_scope.limit(limit)
+      else
+        ancestor_scope
+      end
+    end
+
+    # Returns the projects in `candidates` that have at least one visible, active descendant.
+    # Used to decide whether a node gets an expand arrow or renders as a leaf
+    # has_subprojects?/leaf? is not enough, as it only looks at lft/rgt and would count an archived or invisible
+    # descendant as "has children" too.
+    def having_visible_descendants(candidates)
+      candidates = Array(candidates)
+      return none if candidates.empty?
+
+      visible_ids = visible_active_ids_within(candidates)
+      return none if visible_ids.empty?
+
+      Project
+        .where(id: candidates.map(&:id))
+        .where(
+          "EXISTS (
+           SELECT 1 FROM projects descendant
+           WHERE descendant.id IN (?)
+             AND descendant.lft > projects.lft AND descendant.rgt < projects.rgt
+         )", visible_ids
+        )
+    end
+
     private
 
     def sort_by_name(project_hashes)
       project_hashes.sort_by { |h| h[:project].name&.downcase }
     end
+
+    def visible_active_ids_within(candidates)
+      min_lft = candidates.map(&:lft).min
+      max_rgt = candidates.map(&:rgt).max
+      Project.visible.active.where("lft > ? AND rgt < ?", min_lft, max_rgt).pluck(:id)
+    end
   end
 
   included do

app/views/header/projects/children.html_fragment.erb

@@ -0,0 +1,40 @@
+<%#-- copyright
+OpenProject is an open source project management software.
+Copyright (C) the OpenProject GmbH
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License version 3.
+
+OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+Copyright (C) 2006-2013 Jean-Philippe Lang
+Copyright (C) 2010-2013 the ChiliProject Team
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+See COPYRIGHT and LICENSE files for more details.
+
+++#%>
+
+<%= render(Primer::Alpha::TreeView::SubTree.new(path: @path, node_variant: :anchor)) do |sub| %>
+  <% @children_nodes.each do |node| %>
+    <%= render Header::Projects::NodeComponent.new(
+          component: sub,
+          node:,
+          current_project_id: @current_project_id,
+          favorited_ids: @favorited_ids,
+          jump: @jump
+        ) %>
+  <% end %>
+<% end %>

config/routes.rb

@@ -514,6 +514,7 @@
     resources :projects, only: :index do
       collection do
         get :frame
+        get :children
       end
     end
   end