diff --git a/client/src/api/analytics/analyticsRequest.ts b/client/src/api/analytics/analyticsRequest.ts index 8c4115e40..c3e74670b 100644 --- a/client/src/api/analytics/analyticsRequest.ts +++ b/client/src/api/analytics/analyticsRequest.ts @@ -6,15 +6,17 @@ import { CommonApiParams, toQueryParams } from "./endpoints/types"; /** * The analytics read layer, as a value. * - * Every analytics endpoint is `/sites/:site/` carrying the shared time - * window and filters on the query string plus a handful of endpoint params. + * Analytics endpoints carry the shared time window and filters on the query + * string plus a handful of endpoint params. * A descriptor says only what differs between them; `buildAnalyticsRequest` * turns it into the exact request that goes on the wire, and the query key is * built from that same object — so key and request cannot drift. */ export interface AnalyticsDescriptor { - /** Path under `/sites/:site` — e.g. "overview", "events/names". */ + /** Path under the Site or organization — e.g. "overview", "site-cards-lite". */ path: string; + /** Organization-scoped analytics, instead of the default `/sites/:site` scope. */ + organizationId?: string; /** * Endpoint params beyond the shared time/filter context, already named as * the API expects them on the wire. Undefined values are dropped. @@ -44,6 +46,7 @@ export interface AnalyticsContext { export interface AnalyticsRequest { path: string; + organizationId?: string; params: Record; body?: unknown; unwrap: boolean; @@ -64,6 +67,7 @@ export function buildAnalyticsRequest(descriptor: AnalyticsDescriptor, context: return { path: descriptor.path, + ...(descriptor.organizationId !== undefined ? { organizationId: descriptor.organizationId } : {}), params: omitUndefined({ ...contextParams, ...descriptor.params }), body: descriptor.body?.(common), unwrap: descriptor.unwrap ?? true, @@ -75,8 +79,12 @@ export function buildAnalyticsRequest(descriptor: AnalyticsDescriptor, context: * callers (CSV export), so both go through the same seam. */ export async function fetchAnalytics(site: number | string, request: AnalyticsRequest): Promise { + const base = + request.organizationId === undefined + ? `/sites/${site}` + : `/organizations/${encodeURIComponent(request.organizationId)}`; const response = await authedFetch( - `/sites/${site}/${request.path}`, + `${base}/${request.path}`, request.params, request.body === undefined ? {} : { method: "POST", data: request.body } ); diff --git a/client/src/api/analytics/endpoints/siteCards.ts b/client/src/api/analytics/endpoints/siteCards.ts new file mode 100644 index 000000000..1f9bbfdc3 --- /dev/null +++ b/client/src/api/analytics/endpoints/siteCards.ts @@ -0,0 +1,7 @@ +export interface SiteCardMetrics { + current: { sessions: number; users: number }; + previous: { sessions: number; users: number } | null; + series: { time: string; sessions: number }[]; +} + +export type SiteCardsResponse = Record; diff --git a/client/src/api/analytics/hooks/useGetSiteCards.ts b/client/src/api/analytics/hooks/useGetSiteCards.ts new file mode 100644 index 000000000..6c06e6522 --- /dev/null +++ b/client/src/api/analytics/hooks/useGetSiteCards.ts @@ -0,0 +1,22 @@ +import { useStore } from "@/lib/store"; +import { buildAnalyticsRequest } from "../analyticsRequest"; +import { SiteCardsResponse } from "../endpoints/siteCards"; +import { useAnalyticsContext, useAnalyticsQuery } from "../useAnalyticsQuery"; + +export function useGetSiteCards(organizationId: string, siteIds: number[], lite: boolean) { + const bucket = useStore(state => state.bucket); + const previous = useAnalyticsContext({ periodTime: "previous", useFilters: false }); + const comparison = previous.hasPeriod ? buildAnalyticsRequest({ path: "" }, previous.context).params : null; + + return useAnalyticsQuery({ + key: "site-cards", + path: lite ? "site-cards-lite" : "site-cards", + organizationId, + useFilters: false, + params: { bucket }, + body: () => ({ siteIds: [...new Set(siteIds)].sort((a, b) => a - b), comparison }), + enabled: siteIds.length > 0, + // A new page, organization or period must never show the old cards' totals. + placeholder: false, + }); +} diff --git a/client/src/api/analytics/useAnalyticsQuery.ts b/client/src/api/analytics/useAnalyticsQuery.ts index 29f01b163..b0df8cc3d 100644 --- a/client/src/api/analytics/useAnalyticsQuery.ts +++ b/client/src/api/analytics/useAnalyticsQuery.ts @@ -97,7 +97,10 @@ export function useAnalyticsContext(options: AnalyticsContextOptions = {}): { function useAnalyticsRequest(options: AnalyticsContextOptions & AnalyticsDescriptor) { const { site, context, hasPeriod } = useAnalyticsContext(options); - return { site, hasPeriod, request: buildAnalyticsRequest(options, context) }; + const request = buildAnalyticsRequest(options, context); + const scope = request.organizationId === undefined ? site : `organization:${request.organizationId}`; + const hasScope = request.organizationId === undefined ? !!site : !!request.organizationId; + return { site, scope, hasScope, hasPeriod, request }; } export interface AnalyticsQueryOptions extends AnalyticsContextOptions, AnalyticsDescriptor { @@ -115,24 +118,24 @@ export interface AnalyticsQueryOptions extends AnalyticsContextOptions, A const buildQueryKey = ( key: string | readonly unknown[], - site: number | string | undefined, + scope: number | string | undefined, request: AnalyticsRequest -) => [...(Array.isArray(key) ? key : [key]), site, request.path, request.params, request.body]; +) => [...(Array.isArray(key) ? key : [key]), scope, request.path, request.params, request.body]; export function useAnalyticsQuery(options: AnalyticsQueryOptions): UseQueryResult { - const { site, request, hasPeriod } = useAnalyticsRequest(options); + const { site, scope, hasScope, request, hasPeriod } = useAnalyticsRequest(options); return useQuery({ - queryKey: buildQueryKey(options.key, site, request), + queryKey: buildQueryKey(options.key, scope, request), queryFn: () => fetchAnalytics(site!, request), staleTime: options.staleTime ?? 60_000, refetchInterval: options.refetchInterval, placeholderData: (options.placeholder ?? true) ? (previousData, previousQuery) => - site !== undefined && previousQuery?.queryKey?.includes(site) ? previousData : undefined + scope !== undefined && previousQuery?.queryKey?.includes(scope) ? previousData : undefined : undefined, - enabled: (options.enabled ?? true) && !!site && hasPeriod, + enabled: (options.enabled ?? true) && hasScope && hasPeriod, ...options.props, }); } @@ -153,10 +156,10 @@ export interface AnalyticsInfiniteQueryOptions extends Analytics export function useAnalyticsInfiniteQuery( options: AnalyticsInfiniteQueryOptions ): UseInfiniteQueryResult> { - const { site, request, hasPeriod } = useAnalyticsRequest(options); + const { site, scope, hasScope, request, hasPeriod } = useAnalyticsRequest(options); return useInfiniteQuery, readonly unknown[], TCursor>({ - queryKey: [...buildQueryKey(options.key, site, request), "infinite"], + queryKey: [...buildQueryKey(options.key, scope, request), "infinite"], queryFn: ({ pageParam }) => fetchAnalytics(site!, { ...request, @@ -167,7 +170,7 @@ export function useAnalyticsInfiniteQuery( staleTime: options.staleTime ?? 60_000, refetchInterval: options.refetchInterval, refetchOnWindowFocus: options.refetchOnWindowFocus, - enabled: (options.enabled ?? true) && !!site && hasPeriod, + enabled: (options.enabled ?? true) && hasScope && hasPeriod, }); } diff --git a/client/src/app/(home)/SiteCard.tsx b/client/src/app/(home)/SiteCard.tsx index f623dc977..f95e160ff 100644 --- a/client/src/app/(home)/SiteCard.tsx +++ b/client/src/app/(home)/SiteCard.tsx @@ -1,7 +1,8 @@ import { Tag, Settings } from "lucide-react"; import { useExtracted } from "next-intl"; import Link from "next/link"; -import { useRef } from "react"; +import { Ref, useRef } from "react"; +import { SiteCardMetrics } from "@/api/analytics/endpoints/siteCards"; import { useGetOverview } from "@/api/analytics/hooks/useGetOverview"; import { useGetOverviewBucketed } from "@/api/analytics/hooks/useGetOverviewBucketed"; import { ChangePercentage } from "@/app/[site]/main/components/MainSection/Overview"; @@ -17,7 +18,7 @@ import { LITE_DASHBOARD } from "@/lib/const"; import { useStore } from "@/lib/store"; import { formatter } from "@/lib/utils"; -interface SiteCardProps { +export interface SiteCardProps { siteId: number; name: string; domain: string; @@ -28,17 +29,8 @@ interface SiteCardProps { onTagClick?: (tag: string) => void; } -export function SiteCard({ - siteId, - name, - domain, - tags = [], - allTags = [], - onTagsUpdated, - selectedTags = [], - onTagClick, -}: SiteCardProps) { - const t = useExtracted(); +export function SiteCard(props: SiteCardProps) { + const { siteId } = props; const { ref, isInView } = useInView({ // Start loading slightly before the card comes into view rootMargin: "200px", @@ -84,15 +76,64 @@ export function SiteCard({ hasLoadedData.current = true; } - const hasData = (overviewData?.sessions || 0) > 0; - // Show skeleton when loading or not yet in view, but not if we've already loaded data previously const showSkeleton = (isLoading || isOverviewLoading || !isInView) && !hasLoadedData.current; + return ( + + ); +} + +export function BatchedSiteCard({ metrics, ...props }: SiteCardProps & { metrics?: SiteCardMetrics }) { + const { ref, isInView } = useInView({ rootMargin: "200px", persistVisibility: true }); + return ( + + ); +} + +function SiteCardView({ + siteId, + name, + domain, + tags = [], + allTags = [], + onTagsUpdated, + onTagClick, + cardRef, + data, + overviewData, + overviewDataPrevious, + showSkeleton, + showChart = true, +}: SiteCardProps & { + cardRef: Ref; + data?: SiteCardMetrics["series"]; + overviewData?: SiteCardMetrics["current"]; + overviewDataPrevious?: SiteCardMetrics["previous"]; + showSkeleton: boolean; + showChart?: boolean; +}) { + const t = useExtracted(); + const hasData = (overviewData?.sessions || 0) > 0; return (
{showSkeleton ? ( @@ -124,6 +165,7 @@ export function SiteCard({
- + {showChart && } {!hasData && (
{t("No data available")} @@ -190,10 +232,7 @@ export function SiteCard({
{formatter(overviewData?.sessions ?? 0)}{" "} {overviewData?.sessions && overviewDataPrevious?.sessions ? ( - + ) : null}
@@ -203,10 +242,7 @@ export function SiteCard({
{formatter(overviewData?.users ?? 0)}{" "} {overviewData?.users && overviewDataPrevious?.users ? ( - + ) : null}
diff --git a/client/src/app/(home)/SiteCards.test.tsx b/client/src/app/(home)/SiteCards.test.tsx new file mode 100644 index 000000000..3507d48a3 --- /dev/null +++ b/client/src/app/(home)/SiteCards.test.tsx @@ -0,0 +1,263 @@ +import React from "react"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useStore } from "@/lib/store"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { SiteCards } from "./SiteCards"; + +const mocks = vi.hoisted(() => ({ fetch: vi.fn(), inView: true, lite: true })); +vi.mock("@/api/utils", async importOriginal => ({ + ...(await importOriginal()), + authedFetch: mocks.fetch, +})); +vi.mock("@/lib/const", async importOriginal => ({ + ...(await importOriginal()), + get LITE_DASHBOARD() { + return mocks.lite; + }, +})); +vi.mock("next-intl", () => ({ useExtracted: () => (text: string) => text })); +vi.mock("@/hooks/useInView", () => ({ useInView: () => ({ ref: undefined, isInView: mocks.inView }) })); +vi.mock("@/components/Favicon", () => ({ Favicon: () => null })); +vi.mock("@/components/SiteSettings/SiteSettings", () => ({ SiteSettings: () => null })); +vi.mock("@/components/TagEditor", () => ({ TagEditor: () => null })); +vi.mock("@/components/SiteSessionChart", () => ({ SiteSessionChart: () =>
})); +vi.mock("@/app/[site]/main/components/MainSection/Overview", () => ({ ChangePercentage: () => change })); + +const sites = Array.from({ length: 20 }, (_, i) => ({ siteId: i + 1, name: `Site ${i + 1}`, domain: "example.com" })); +let client: QueryClient; + +beforeEach(() => { + mocks.fetch.mockReset(); + mocks.inView = true; + mocks.lite = true; + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + useStore.setState({ + site: "", + time: { mode: "past-minutes", pastMinutesStart: 1440, pastMinutesEnd: 0 }, + previousTime: { mode: "past-minutes", pastMinutesStart: 2880, pastMinutesEnd: 1440 }, + bucket: "hour", + timezone: "America/New_York", + filters: [{ parameter: "country", type: "equals", value: ["CA"] }], + }); + mocks.fetch.mockImplementation( + async (path: string, _params: unknown, config?: { data: { siteIds: number[]; comparison: unknown } }) => { + if (path.endsWith("/site-cards-lite") || path.endsWith("/site-cards")) { + return { + data: Object.fromEntries( + config!.data.siteIds.map(siteId => [ + siteId, + { + current: { sessions: 123, users: 45 }, + previous: config!.data.comparison === null ? null : { sessions: 100, users: 40 }, + series: [{ time: "2026-09-20 12:00:00", sessions: 123 }], + }, + ]) + ), + }; + } + return { data: path.includes("bucketed") ? [] : { sessions: 123, users: 45 } }; + } + ); +}); + +afterEach(() => { + cleanup(); + client.clear(); +}); + +function cards(organizationId = "org-1", pageSites = sites) { + return ( + + + + + + ); +} + +describe("homepage Site cards", () => { + it("loads 20 cards with one analytics request including both periods", async () => { + render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + expect(mocks.fetch).toHaveBeenCalledWith( + "/organizations/org-1/site-cards-lite", + { + time_zone: "America/New_York", + past_minutes_start: 1440, + past_minutes_end: 0, + bucket: "hour", + }, + { + method: "POST", + data: { + siteIds: sites.map(site => site.siteId), + comparison: { time_zone: "America/New_York", past_minutes_start: 2880, past_minutes_end: 1440 }, + }, + } + ); + }); + + it("shows totals off screen and scrolling mounts charts without more requests", async () => { + mocks.inView = false; + const view = render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(screen.queryAllByTestId("chart")).toHaveLength(0); + mocks.inView = true; + view.rerender(cards()); + expect(screen.getAllByTestId("chart")).toHaveLength(20); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + }); + + it("sends no comparison window or percentages when comparison is disabled", async () => { + render(cards()); + await waitFor(() => expect(screen.getAllByText("change")).toHaveLength(40)); + act(() => useStore.setState({ previousTime: null })); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2)); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch.mock.lastCall![2].data.comparison).toBeNull(); + expect(screen.queryAllByText("change")).toHaveLength(0); + }); + + it("loads just the new page's sites and isolates the organization cache", async () => { + const view = render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + const nextSites = [{ siteId: 21, name: "Site 21", domain: "example.com" }]; + view.rerender(cards("org-1", nextSites)); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2)); + expect(mocks.fetch.mock.lastCall![2].data.siteIds).toEqual([21]); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(1)); + + // Same site IDs, new org: an in-flight request must not display the old + // organization's cached totals, even while a transfer is being resolved. + mocks.fetch.mockImplementationOnce(() => new Promise(() => {})); + view.rerender(cards("org-2", nextSites)); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(3)); + expect(mocks.fetch.mock.lastCall![0]).toBe("/organizations/org-2/site-cards-lite"); + expect(screen.queryAllByText("123")).toHaveLength(0); + }); + + it("keys requests by the current period, comparison, timezone and bucket", async () => { + render(cards()); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(1)); + act(() => useStore.setState({ time: { mode: "day", day: "2026-09-18" } })); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2)); + expect(mocks.fetch.mock.lastCall![1]).toMatchObject({ start_date: "2026-09-18", end_date: "2026-09-18" }); + act(() => useStore.setState({ previousTime: { mode: "day", day: "2025-09-18" } })); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(3)); + expect(mocks.fetch.mock.lastCall![2].data.comparison.start_date).toBe("2025-09-18"); + act(() => useStore.setState({ timezone: "Asia/Kolkata" })); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(4)); + expect(mocks.fetch.mock.lastCall![1].time_zone).toBe("Asia/Kolkata"); + act(() => useStore.setState({ bucket: "day" })); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(5)); + expect(mocks.fetch.mock.lastCall![1].bucket).toBe("day"); + }); + + it("reuses the batch for a reorder and ignores per-site dashboard filters", async () => { + const view = render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + view.rerender(cards("org-1", [...sites].reverse())); + act(() => useStore.setState({ filters: [], site: "42" })); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + }); + + it.each(["current", "previous"])("retains the existing fallback for exact %s windows", async period => { + const range = { + mode: "range" as const, + startDate: "2026-09-18", + endDate: "2026-09-18", + startTime: "10:30", + endTime: "12:45", + }; + useStore.setState(period === "current" ? { time: range } : { previousTime: range }); + render(cards()); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(60)); + expect(mocks.fetch.mock.calls.every(([path]) => path.startsWith("/sites/"))).toBe(true); + expect(mocks.fetch.mock.calls.some(([, params]) => params.start_datetime)).toBe(true); + }); + + it("batches standard deployments without using the materialized-view endpoint", async () => { + mocks.lite = false; + render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + expect(mocks.fetch.mock.lastCall![0]).toBe("/organizations/org-1/site-cards"); + expect(mocks.fetch.mock.lastCall![2].data.siteIds).toEqual(sites.map(site => site.siteId)); + }); + + it("batches exact datetime ranges with minute buckets in standard mode", async () => { + mocks.lite = false; + useStore.setState({ + time: { mode: "range", startDate: "2026-09-18", endDate: "2026-09-18", startTime: "10:30", endTime: "12:45" }, + previousTime: { + mode: "range", + startDate: "2026-09-17", + endDate: "2026-09-17", + startTime: "10:30", + endTime: "12:45", + }, + bucket: "minute", + }); + render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + expect(mocks.fetch.mock.lastCall![0]).toBe("/organizations/org-1/site-cards"); + expect(mocks.fetch.mock.lastCall![1]).toMatchObject({ + start_datetime: "2026-09-18 14:30:00", + end_datetime: "2026-09-18 16:45:00", + bucket: "minute", + }); + expect(mocks.fetch.mock.lastCall![2].data.comparison).toMatchObject({ + start_datetime: "2026-09-17 14:30:00", + end_datetime: "2026-09-17 16:45:00", + }); + }); + + it("batches all-time hourly charts in standard mode", async () => { + mocks.lite = false; + useStore.setState({ time: { mode: "all-time" }, previousTime: null, bucket: "hour" }); + render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + expect(mocks.fetch.mock.lastCall![0]).toBe("/organizations/org-1/site-cards"); + expect(mocks.fetch.mock.lastCall![2].data.comparison).toBeNull(); + }); + + it("does not reuse MV metrics for a raw-events request", async () => { + const view = render(cards()); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + mocks.lite = false; + mocks.fetch.mockImplementationOnce(() => new Promise(() => {})); + view.rerender(cards()); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(2)); + expect(mocks.fetch.mock.lastCall![0]).toBe("/organizations/org-1/site-cards"); + expect(screen.queryAllByText("123")).toHaveLength(0); + }); + + it("retains event-only buckets in unbounded hourly charts through the existing endpoint", async () => { + useStore.setState({ time: { mode: "all-time" }, previousTime: null, bucket: "hour" }); + render(cards()); + await waitFor(() => expect(mocks.fetch).toHaveBeenCalledTimes(40)); + expect(mocks.fetch.mock.calls.every(([path]) => path.startsWith("/sites/"))).toBe(true); + }); + + it("makes no analytics request for an empty page or missing organization", () => { + const view = render(cards("org-1", [])); + expect(mocks.fetch).not.toHaveBeenCalled(); + view.rerender(cards("", sites)); + expect(mocks.fetch).not.toHaveBeenCalled(); + }); + + it("shows an error and retries the batch without displaying zero metrics", async () => { + mocks.fetch.mockRejectedValueOnce(new Error("ClickHouse unavailable")); + render(cards()); + await waitFor(() => expect(screen.getByText("Failed to load data")).toBeTruthy()); + expect(screen.queryAllByText("0")).toHaveLength(0); + fireEvent.click(screen.getByRole("button", { name: "Try Again" })); + await waitFor(() => expect(screen.getAllByText("123")).toHaveLength(20)); + expect(mocks.fetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/client/src/app/(home)/SiteCards.tsx b/client/src/app/(home)/SiteCards.tsx new file mode 100644 index 000000000..281ba4532 --- /dev/null +++ b/client/src/app/(home)/SiteCards.tsx @@ -0,0 +1,41 @@ +import { useGetSiteCards } from "@/api/analytics/hooks/useGetSiteCards"; +import { ErrorState } from "@/components/ErrorState"; +import { LITE_DASHBOARD } from "@/lib/const"; +import { useStore } from "@/lib/store"; +import { hasRangeTimes } from "@/lib/time"; +import { BatchedSiteCard, SiteCard, SiteCardProps } from "./SiteCard"; + +interface SiteCardsProps extends Omit { + organizationId: string; + sites: Pick[]; +} + +export function SiteCards({ organizationId, sites, ...props }: SiteCardsProps) { + const time = useStore(state => state.time); + const previousTime = useStore(state => state.previousTime); + const bucket = useStore(state => state.bucket); + if (!LITE_DASHBOARD) { + return ; + } + // An unbounded hourly chart also contains event-only buckets from the old + // overview JOIN. Keep that path; bounded charts get those zeros from FILL. + const unboundedHourlyChart = time.mode === "all-time" && !["day", "week", "month", "year"].includes(bucket); + // Exact datetime ranges retain the existing raw-events fallback. A custom + // comparison can also be exact even when the current period is a whole day. + if (!unboundedHourlyChart && !hasRangeTimes(time) && !(previousTime && hasRangeTimes(previousTime))) { + return ; + } + return sites.map(site => ); +} + +function BatchedSiteCards({ organizationId, sites, lite, ...props }: SiteCardsProps & { lite: boolean }) { + const { data, error, refetch } = useGetSiteCards( + organizationId, + sites.map(site => site.siteId), + lite + ); + if (error && !data) { + return ; + } + return sites.map(site => ); +} diff --git a/client/src/app/(home)/page.tsx b/client/src/app/(home)/page.tsx index bc45089cb..ea383d1e5 100644 --- a/client/src/app/(home)/page.tsx +++ b/client/src/app/(home)/page.tsx @@ -24,11 +24,9 @@ import { useSetPageTitle } from "../../hooks/useSetPageTitle"; import { authClient } from "../../lib/auth"; import { canGoBack, canGoForward, goBack, goForward, useStore } from "../../lib/store"; import { AddSite } from "../components/AddSite"; -import { SiteCard } from "./SiteCard"; +import { SiteCards } from "./SiteCards"; -// Only render a bounded slice of site cards at a time. Each card mounts an -// IntersectionObserver and fires its own analytics queries, so rendering every -// site at once crashes orgs with thousands of websites. +// Bound both rendering and the lite analytics batch for large organizations. const PAGE_SIZE = 20; export default function Home() { @@ -194,21 +192,14 @@ export default function Home() { const siteCards = (
- {paginatedSites?.map(site => { - return ( - - ); - })} + ({ ...site, tags: site.tags ?? [] })) ?? []} + allTags={allTags} + onTagsUpdated={refetchSites} + selectedTags={selectedTags} + onTagClick={handleTagClick} + /> {hasSites && hasNoMatches ? ( {t("No matching websites")} diff --git a/client/src/components/SiteSessionChart.tsx b/client/src/components/SiteSessionChart.tsx index fef124ad5..4e4e00650 100644 --- a/client/src/components/SiteSessionChart.tsx +++ b/client/src/components/SiteSessionChart.tsx @@ -9,7 +9,7 @@ import { useStore } from "../lib/store"; import { ChartTooltip } from "./charts/ChartTooltip"; interface SiteSessionChartProps { - data: GetOverviewBucketedResponse; + data: Pick[]; } export function SiteSessionChart({ data }: SiteSessionChartProps) { diff --git a/server/src/api/analytics/getSiteCards.test.ts b/server/src/api/analytics/getSiteCards.test.ts new file mode 100644 index 000000000..935bbf6c5 --- /dev/null +++ b/server/src/api/analytics/getSiteCards.test.ts @@ -0,0 +1,220 @@ +import Fastify, { FastifyInstance } from "fastify"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + checkApiKey: vi.fn(), + getUserIsInOrg: vi.fn(), + getSessionFromReq: vi.fn(), + getSitesUserHasAccessTo: vi.fn(), + siteIdsInOrganization: vi.fn(), + query: vi.fn(), +})); +vi.mock("../../lib/auth-utils.js", () => mocks); +vi.mock("../../lib/access.js", () => ({ ...mocks })); +vi.mock("../../lib/siteConfig.js", () => ({ siteConfig: {} })); +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ clickhouse: { query: mocks.query } })); + +import { requireOrgMember } from "../../lib/auth-middleware.js"; +import { getSiteCards, getSiteCardsLite } from "./getSiteCards.js"; + +let app: FastifyInstance; +const allIds = Array.from({ length: 20 }, (_, i) => i + 1); +const comparison = { past_minutes_start: 2880, past_minutes_end: 1440, time_zone: "America/New_York" }; + +beforeEach(async () => { + vi.resetAllMocks(); + vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-09-20T20:41:33Z")); + mocks.checkApiKey.mockResolvedValue({ valid: false, role: null, statements: null }); + mocks.getUserIsInOrg.mockResolvedValue(true); + mocks.getSessionFromReq.mockResolvedValue({ user: { id: "user-1" } }); + mocks.getSitesUserHasAccessTo.mockResolvedValue(allIds.map(siteId => ({ siteId, organizationId: "org-1" }))); + mocks.siteIdsInOrganization.mockImplementation(async ids => ids); + mocks.query.mockImplementation(async ({ query }: { query: string }) => ({ + json: async () => + query.includes("AS current_sessions") + ? [ + { + site_id: 1, + current_sessions: "12", + current_users: "8", + previous_sessions: "10", + previous_users: "7", + }, + ] + : [ + { + time: "2026-09-19 16:00:00", + site_sessions: [ + [1, "3"], + [2, "9"], + ], + }, + { time: "2026-09-19 17:00:00", site_sessions: [] }, + { time: "2026-09-19 18:00:00", site_sessions: [[1, "9"]] }, + ], + })); + app = Fastify(); + app.post<{ Params: { organizationId: string }; Querystring: unknown; Body: unknown }>( + "/organizations/:organizationId/site-cards-lite", + { + preHandler: [requireOrgMember({ resource: "analytics", action: "read" })], + }, + getSiteCardsLite + ); + app.post<{ Params: { organizationId: string }; Querystring: unknown; Body: unknown }>( + "/organizations/:organizationId/site-cards", + { preHandler: [requireOrgMember({ resource: "analytics", action: "read" })] }, + getSiteCards + ); + await app.ready(); +}); + +afterEach(async () => { + await app.close(); + vi.restoreAllMocks(); +}); + +describe.each(["site-cards", "site-cards-lite"])("batched %s", endpoint => { + const url = `/organizations/org-1/${endpoint}?past_minutes_start=1440&past_minutes_end=0&time_zone=America%2FNew_York&bucket=hour`; + const request = (payload: unknown = { siteIds: allIds, comparison }, requestUrl = url) => + app.inject({ + method: "POST", + url: requestUrl, + payload: JSON.stringify(payload), + headers: { "content-type": "application/json" }, + }); + + it("runs exactly two bounded queries for 20 sites, returning totals and zero-filled sessions", async () => { + const response = await request(); + expect(response.statusCode).toBe(200); + const data = response.json().data; + expect(Object.keys(data)).toHaveLength(20); + expect(data[1]).toEqual({ + current: { sessions: 12, users: 8 }, + previous: { sessions: 10, users: 7 }, + series: [ + { time: "2026-09-19 16:00:00", sessions: 3 }, + { time: "2026-09-19 17:00:00", sessions: 0 }, + { time: "2026-09-19 18:00:00", sessions: 9 }, + ], + }); + expect(data[20]).toEqual({ + current: { sessions: 0, users: 0 }, + previous: { sessions: 0, users: 0 }, + series: data[1].series.map((point: { time: string }) => ({ time: point.time, sessions: 0 })), + }); + expect(mocks.query).toHaveBeenCalledTimes(2); + for (const [spec] of mocks.query.mock.calls) { + expect(spec.query_params).toEqual({ siteIds: allIds }); + expect(spec.clickhouse_settings.max_execution_time).toBe(60); + if (endpoint === "site-cards") { + expect(spec.query).toContain("FROM events"); + expect(spec.query).not.toContain("_mv_target"); + } + } + }); + + it("omits the previous period when comparison is off and deduplicates IDs", async () => { + const response = await request({ siteIds: [1, 1], comparison: null }); + expect(response.statusCode).toBe(200); + expect(Object.keys(response.json().data)).toEqual(["1"]); + expect(response.json().data[1].previous).toBeNull(); + expect(mocks.query.mock.calls[0][0].query_params.siteIds).toEqual([1]); + expect(mocks.query.mock.calls[0][0].query).not.toContain("2026-09-18"); + }); + + it.each([ + { siteIds: [], comparison }, + { siteIds: [...allIds, 21], comparison }, + { siteIds: [-1], comparison }, + { siteIds: [1.5], comparison }, + { siteIds: ["1) OR 1"], comparison }, + { siteIds: [1] }, + { siteIds: [1], comparison: { start_date: "2026-09-18" } }, + { siteIds: [1], comparison: { past_minutes_start: 60, past_minutes_end: 60 } }, + { siteIds: [1], comparison: { start_date: "2026-09-19", end_date: "2026-09-18" } }, + { siteIds: [1], comparison: { start_datetime: "2026-09-18 10:30:00" } }, + { siteIds: [1], comparison: { start_datetime: "2026-09-18 11:30:00", end_datetime: "2026-09-18 10:30:00" } }, + ])("rejects malformed or unsupported bodies without querying analytics: %j", async body => { + expect((await request(body)).statusCode).toBe(400); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it.each([ + "past_minutes_start=1440", + "past_minutes_start=Infinity&past_minutes_end=0", + "past_minutes_start=&past_minutes_end=", + "start_date=bad&end_date=bad", + "time_zone=invalid", + "bucket=bad", + "filters=[]", + "start_datetime=bad&end_datetime=bad", + ])("rejects invalid current windows or unsupported parameters: %s", async query => { + const response = await request(undefined, `/organizations/org-1/${endpoint}?${query}`); + expect(response.statusCode).toBe(400); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it("accepts the dashboard's empty date bounds for all-time", async () => { + const response = await request( + { siteIds: [1, 20], comparison: null }, + `/organizations/org-1/${endpoint}?start_date=&end_date=&bucket=month` + ); + expect(response.statusCode).toBe(200); + expect(response.json().data[1].series).toHaveLength(2); + expect(response.json().data[20].series).toEqual([]); + }); + + it("rejects a site outside the organization even if the user can access it", async () => { + mocks.siteIdsInOrganization.mockResolvedValue([1]); + expect((await request({ siteIds: [1, 2], comparison })).statusCode).toBe(403); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it("rejects sites excluded by member or team access before querying ClickHouse", async () => { + mocks.getSitesUserHasAccessTo.mockResolvedValue([{ siteId: 1, organizationId: "org-1" }]); + expect((await request({ siteIds: [1, 2], comparison })).statusCode).toBe(403); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it("rejects unauthenticated callers and keys without analytics:read", async () => { + mocks.getUserIsInOrg.mockResolvedValue(false); + mocks.getSessionFromReq.mockResolvedValue(null); + expect((await request()).statusCode).toBe(403); + mocks.checkApiKey.mockResolvedValue({ valid: true, userId: "user-1", statements: { org: ["read"] } }); + const denied = await request(); + expect(denied.statusCode).toBe(403); + expect(denied.json().required).toBe("analytics:read"); + expect(mocks.query).not.toHaveBeenCalled(); + }); + + it("supports organization-owned analytics keys without a session user", async () => { + mocks.getUserIsInOrg.mockResolvedValue(false); + mocks.getSessionFromReq.mockResolvedValue(null); + mocks.checkApiKey.mockResolvedValue({ valid: true, organizationId: "org-1", statements: { analytics: ["read"] } }); + expect((await request()).statusCode).toBe(200); + expect(mocks.getSitesUserHasAccessTo.mock.calls[0][0].apiKeyOrganizationId).toBe("org-1"); + }); + + it("reports query failures as errors, not zero totals", async () => { + mocks.query.mockRejectedValue(new Error("ClickHouse unavailable")); + const response = await request(); + expect(response.statusCode).toBe(500); + expect(response.json()).toEqual({ error: "Failed to fetch site cards" }); + }); + + it("serves exact datetime windows only from raw events", async () => { + const response = await request( + { siteIds: [1], comparison: { start_datetime: "2026-09-17 10:30:00", end_datetime: "2026-09-17 11:30:00" } }, + `/organizations/org-1/${endpoint}?start_datetime=2026-09-18+10:30:00&end_datetime=2026-09-18+11:30:00&bucket=minute` + ); + expect(response.statusCode).toBe(endpoint === "site-cards" ? 200 : 400); + if (endpoint === "site-cards") { + expect(mocks.query.mock.calls[0][0].query).toContain("timestamp >= toDateTime('2026-09-18 10:30:00', 'UTC')"); + expect(mocks.query.mock.calls[0][0].query).toContain("timestamp < toDateTime('2026-09-17 11:30:00', 'UTC')"); + expect(mocks.query.mock.calls[1][0].query).toContain("toStartOfMinute"); + } else { + expect(mocks.query).not.toHaveBeenCalled(); + } + }); +}); diff --git a/server/src/api/analytics/getSiteCards.ts b/server/src/api/analytics/getSiteCards.ts new file mode 100644 index 000000000..613bb8291 --- /dev/null +++ b/server/src/api/analytics/getSiteCards.ts @@ -0,0 +1,135 @@ +import { z } from "zod"; +import { siteIdsInOrganization } from "../../lib/access.js"; +import { getSitesUserHasAccessTo } from "../../lib/auth-utils.js"; +import { buildSiteCardsQueries as buildLiteSiteCardsQueries } from "./lite/siteCardsQuery.js"; +import { hasLiteDatetimeRange } from "./lite/utils.js"; +import { analyticsRoute, runAnalyticsQuery } from "./utils/analyticsQuery.js"; +import { validateHttpTimeParams } from "./utils/query-validation.js"; +import { TimeWindowParams } from "./utils/timeWindow.js"; +import { buildSiteCardsQueries } from "./siteCardsQuery.js"; + +const minutesSchema = z + .union([z.string().trim().min(1), z.number()]) + .pipe(z.coerce.number().finite().nonnegative()) + .optional(); + +const timeFields = z + .object({ + start_date: z.string().optional(), + end_date: z.string().optional(), + start_datetime: z.string().optional(), + end_datetime: z.string().optional(), + time_zone: z.string().optional(), + past_minutes_start: minutesSchema, + past_minutes_end: minutesSchema, + }) + .strict(); + +function validateWindow(params: TimeWindowParams, ctx: z.RefinementCtx) { + const error = validateHttpTimeParams(params); + if (error) ctx.addIssue({ code: "custom", message: error }); + if (params.start_date && params.end_date && params.start_date > params.end_date) { + ctx.addIssue({ code: "custom", message: "start_date must not be after end_date" }); + } +} + +// The homepage has no dimension filters. Reject them instead of silently +// answering an unfiltered question. +const querySchema = timeFields + .extend({ + bucket: z + .enum(["minute", "five_minutes", "ten_minutes", "fifteen_minutes", "hour", "day", "week", "month", "year"]) + .default("hour"), + }) + .superRefine(validateWindow); +const bodySchema = z + .object({ + siteIds: z.array(z.number().int().positive().max(2_147_483_647)).min(1).max(20), + comparison: timeFields.superRefine(validateWindow).nullable(), + }) + .strict(); + +interface TotalsRow { + site_id: number; + current_sessions: number; + current_users: number; + previous_sessions: number; + previous_users: number; +} + +interface SeriesRow { + time: string; + // processResults only coerces top-level values; UInt64s in tuples are strings. + site_sessions: [number, number | string][]; +} + +function createSiteCardsHandler(lite: boolean) { + return analyticsRoute<{ + Params: { organizationId: string }; + Querystring: unknown; + Body: unknown; + }>("site cards", async (req, res) => { + const query = querySchema.safeParse(req.query); + const body = bodySchema.safeParse(req.body); + if (!query.success || !body.success) { + return res.status(400).send({ error: "Invalid site card parameters" }); + } + + // The raw path supports exact windows; hourly MVs cannot represent them. + if ( + lite && + (hasLiteDatetimeRange(query.data) || (body.data.comparison && hasLiteDatetimeRange(body.data.comparison))) + ) { + return res.status(400).send({ error: "Exact datetime windows require the standard site card endpoint" }); + } + + const siteIds = [...new Set(body.data.siteIds)]; + const { organizationId } = req.params; + const [accessibleSites, orgSiteIds] = await Promise.all([ + getSitesUserHasAccessTo(req), + siteIdsInOrganization(siteIds, organizationId), + ]); + const accessibleIds = new Set(accessibleSites.map(site => site.siteId)); + if (orgSiteIds.length !== siteIds.length || siteIds.some(id => !accessibleIds.has(id))) { + return res.status(403).send({ error: "Forbidden" }); + } + + const queries = (lite ? buildLiteSiteCardsQueries : buildSiteCardsQueries)({ + siteIds, + current: query.data, + comparison: body.data.comparison, + bucket: query.data.bucket, + }); + const [totals, series] = await Promise.all([ + runAnalyticsQuery(queries.totals), + runAnalyticsQuery(queries.series), + ]); + const totalsBySite = new Map(totals.map(row => [row.site_id, row])); + const buckets = series.map(row => ({ time: row.time, counts: new Map(row.site_sessions) })); + const data = Object.fromEntries( + siteIds.map(siteId => { + const total = totalsBySite.get(siteId); + return [ + siteId, + { + current: { sessions: total?.current_sessions ?? 0, users: total?.current_users ?? 0 }, + previous: + body.data.comparison === null + ? null + : { + sessions: total?.previous_sessions ?? 0, + users: total?.previous_users ?? 0, + }, + series: buckets + .filter(({ counts }) => queries.fillMissingBuckets || counts.has(siteId)) + .map(({ time, counts }) => ({ time, sessions: Number(counts.get(siteId) ?? 0) })), + }, + ]; + }) + ); + return res.send({ data }); + }); +} + +export const getSiteCards = createSiteCardsHandler(false); +export const getSiteCardsLite = createSiteCardsHandler(true); diff --git a/server/src/api/analytics/index.ts b/server/src/api/analytics/index.ts index 8e44749f8..fecbbb5c6 100644 --- a/server/src/api/analytics/index.ts +++ b/server/src/api/analytics/index.ts @@ -65,6 +65,7 @@ export { getOrgEventCount } from "./getOrgEventCount.js"; export { getOverview } from "./getOverview.js"; export { getOverviewBucketed } from "./getOverviewBucketed.js"; export { getOverviewLite } from "./lite/getOverviewLite.js"; +export { getSiteCards, getSiteCardsLite } from "./getSiteCards.js"; export { getOverviewBucketedLite } from "./lite/getOverviewBucketedLite.js"; export { getMetricLite } from "./lite/getMetricLite.js"; export { getPageTitles } from "./getPageTitles.js"; diff --git a/server/src/api/analytics/lite/siteCardsQuery.test.ts b/server/src/api/analytics/lite/siteCardsQuery.test.ts new file mode 100644 index 000000000..9910be2c7 --- /dev/null +++ b/server/src/api/analytics/lite/siteCardsQuery.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildSiteCardsQueries } from "./siteCardsQuery.js"; + +const siteIds = [1, 2, 3]; +const current = { past_minutes_start: 1440, past_minutes_end: 0, time_zone: "America/New_York" }; +const comparison = { ...current, past_minutes_start: 2880, past_minutes_end: 1440 }; +const now = Date.parse("2026-09-20T20:41:33Z"); + +describe("Site card queries", () => { + it("resolves adjacent rolling periods from one instant, regardless of query construction time", () => { + const clock = vi.spyOn(Date, "now").mockReturnValue(now + 3600_000); + try { + const { totals, series } = buildSiteCardsQueries({ siteIds, current, comparison, bucket: "hour" }, now); + expect(totals.query).toContain("session_hour > toDateTime('2026-09-18 20:41:33', 'UTC')"); + expect(totals.query).toContain("session_hour <= toDateTime('2026-09-19 20:41:33', 'UTC')"); + expect(totals.query).toContain("session_hour > toDateTime('2026-09-19 20:41:33', 'UTC')"); + expect(series.query).toContain("start_time <= toDateTime('2026-09-20 20:41:33', 'UTC')"); + expect(clock).not.toHaveBeenCalled(); + } finally { + clock.mockRestore(); + } + }); + + it("bounds both aggregates independently so overlapping or nonadjacent comparisons work", () => { + const { totals } = buildSiteCardsQueries( + { + siteIds, + current, + comparison: { ...comparison, past_minutes_start: 4320, past_minutes_end: 720 }, + bucket: "hour", + }, + now + ); + expect(totals.query).toMatch( + /uniqMergeIf\(users, \(1 AND session_hour > .*session_hour <= .*\)\) AS current_users/ + ); + expect(totals.query).toMatch( + /uniqMergeIf\(users, \(1 AND session_hour > .*session_hour <= .*\)\) AS previous_users/ + ); + expect(totals.query).toContain("2026-09-17 20:41:33"); + expect(totals.query).toContain("2026-09-20 08:41:33"); + }); + + it("includes site_id in session deduplication and avoids work the sparkline does not display", () => { + const { series } = buildSiteCardsQueries({ siteIds, current, comparison, bucket: "hour" }, now); + expect(series.query).toContain("GROUP BY site_id, session_id"); + expect(series.query).toContain("min(start_time) AS session_start"); + expect(series.query).toContain("WITH FILL"); + expect(series.query).not.toMatch(/JOIN|uniqMerge|pageviews|end_time/); + expect(series.params).toEqual({ siteIds }); + }); + + it.each(["day", "week", "month", "year"] as const)("keeps %s charts on the hourly rollup", bucket => { + const { series } = buildSiteCardsQueries({ siteIds, current, comparison, bucket }, now); + expect(series.query).toContain("FROM session_hourly_mv_target"); + expect(series.query).not.toContain("FROM sessions_mv_target"); + expect(series.query).toContain("sum(sessions)"); + }); + + it("promotes minute buckets to hour, matching the existing lite chart", () => { + const args = { siteIds, current, comparison }; + expect(buildSiteCardsQueries({ ...args, bucket: "minute" }, now)).toEqual( + buildSiteCardsQueries({ ...args, bucket: "hour" }, now) + ); + }); + + it("keeps all-time unbounded and unfilled without inventing a comparison", () => { + const { totals, series } = buildSiteCardsQueries({ siteIds, current: {}, comparison: null, bucket: "month" }, now); + expect(totals.query).toContain("sumIf(sessions, 0) AS previous_sessions"); + expect(series.query).not.toContain("WITH FILL"); + expect(series.query).not.toContain("session_hour >"); + }); +}); diff --git a/server/src/api/analytics/lite/siteCardsQuery.ts b/server/src/api/analytics/lite/siteCardsQuery.ts new file mode 100644 index 000000000..0c00f56d4 --- /dev/null +++ b/server/src/api/analytics/lite/siteCardsQuery.ts @@ -0,0 +1,70 @@ +import { SiteCardQueries, SiteCardQueryParams } from "../siteCardsQuery.js"; +import { resolveTimeWindow } from "../utils/timeWindow.js"; +import { liteBucket } from "./utils.js"; + +export function buildSiteCardsQueries( + { siteIds, current, comparison, bucket: requestedBucket }: SiteCardQueryParams, + now = Date.now() +): SiteCardQueries { + // Resolve both periods against one clock, including adjacent rolling windows. + const window = resolveTimeWindow(current, now); + const previous = comparison === null ? null : resolveTimeWindow(comparison, now); + const currentPredicate = `(1 ${window.where("session_hour")})`; + const previousPredicate = previous ? `(1 ${previous.where("session_hour")})` : "0"; + const bucket = liteBucket(requestedBucket); + const params = { siteIds }; + + const totals = { + query: ` + SELECT site_id, + sumIf(sessions, ${currentPredicate}) AS current_sessions, + uniqMergeIf(users, ${currentPredicate}) AS current_users, + sumIf(sessions, ${previousPredicate}) AS previous_sessions, + uniqMergeIf(users, ${previousPredicate}) AS previous_users + FROM session_hourly_mv_target + WHERE site_id IN {siteIds:Array(Int32)} + AND (${currentPredicate} OR ${previousPredicate}) + GROUP BY site_id + `, + params, + }; + + // Match the existing lite chart: live sessions for hours, refreshable rollup + // for day+ buckets. The cards only display sessions, so no events JOIN or + // user-state merge is needed for their sparklines. + const counts = + bucket === "hour" + ? ` + SELECT site_id, ${window.bucketed("session_start", bucket)} AS time, count() AS sessions + FROM ( + SELECT site_id, session_id, min(start_time) AS session_start + FROM sessions_mv_target + WHERE site_id IN {siteIds:Array(Int32)} ${window.where("start_time")} + GROUP BY site_id, session_id + ) + GROUP BY site_id, time + ` + : ` + SELECT site_id, ${window.bucketed("session_hour", bucket)} AS time, sum(sessions) AS sessions + FROM session_hourly_mv_target + WHERE site_id IN {siteIds:Array(Int32)} ${window.where("session_hour")} + GROUP BY site_id, time + `; + + return { + totals, + fillMissingBuckets: !window.isAllTime, + series: { + // Fill the shared time axis once. Empty buckets have an empty array; + // the handler supplies zero for every missing site, including sites with + // no rows anywhere in the selected window. + query: ` + SELECT time, groupArray((site_id, sessions)) AS site_sessions + FROM (${counts}) + GROUP BY time + ORDER BY time ${window.fill(bucket)} + `, + params, + }, + }; +} diff --git a/server/src/api/analytics/siteCardsQuery.clickhouse.test.ts b/server/src/api/analytics/siteCardsQuery.clickhouse.test.ts new file mode 100644 index 000000000..c912e2ff2 --- /dev/null +++ b/server/src/api/analytics/siteCardsQuery.clickhouse.test.ts @@ -0,0 +1,186 @@ +import { createClient } from "@clickhouse/client"; +import { TimeBucket } from "@rybbit/shared"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Run against any ClickHouse using CLICKHOUSE_TEST_URL and optional +// CLICKHOUSE_TEST_USER / CLICKHOUSE_TEST_PASSWORD. Every query supplies its own +// events CTE: no tables, migrations or production data are needed. +vi.mock("../../db/clickhouse/clickhouse.js", () => ({ clickhouse: {} })); + +import { buildOverviewQuery } from "../../services/siteMetrics/siteMetrics.js"; +import { buildOverviewBucketedQuery } from "./getOverviewBucketed.js"; +import { buildSiteCardsQueries } from "./siteCardsQuery.js"; +import { resolveTimeWindow, TimeWindowParams } from "./utils/timeWindow.js"; + +const now = Date.parse("2026-09-20T20:41:33Z"); +const siteIds = [1, 2, 3]; +// site, time, session, fingerprint, identified user, event type +const events = [ + [1, "2026-09-19 19:00:00", "cross-period", "shared-device", "", "pageview"], + [1, "2026-09-19 19:10:00", "previous-visit", "shared-device", "", "pageview"], + [1, "2026-09-19 21:00:00", "cross-period", "shared-device", "alice", "pageview"], + [1, "2026-09-19 21:05:00", "other-person", "shared-device", "bob", "custom_event"], + [1, "2026-09-19 22:00:00", "identify-later", "other-device", "", "pageview"], + [1, "2026-09-19 22:01:00", "identify-later", "other-device", "alice", "custom_event"], + [1, "2026-09-20 01:00:00", "identify-later", "other-device", "alice", "pageview"], + [1, "2026-09-20 20:41:33", "end-boundary", "boundary-device", "", "error"], + // Session IDs can coincide across sites, and no session has multiple identities. + [2, "2026-09-19 21:30:00", "cross-period", "site-two", "charlie", "pageview"], + [2, "2026-09-19 23:30:00", "cross-period", "site-two", "charlie", "pageview"], + [1, "2024-03-10 06:30:00", "spring-a", "spring-a", "", "pageview"], + [2, "2024-03-10 07:30:00", "spring-b", "spring-b", "", "pageview"], + [1, "2024-11-03 05:30:00", "fall-a", "fall-a", "", "pageview"], + [1, "2024-11-03 06:30:00", "fall-b", "fall-b", "", "pageview"], +]; +const fixture = `WITH events AS (SELECT * FROM values( + 'site_id Int32, timestamp DateTime, session_id String, user_id String, identified_user_id String, type String', + ${events.map(row => `(${row.map(value => (typeof value === "number" ? value : `'${value}'`)).join(",")})`).join(",")} +))`; +const day = (date: string, time_zone = "America/New_York") => ({ start_date: date, end_date: date, time_zone }); +const rolling = { past_minutes_start: 1440, past_minutes_end: 0, time_zone: "America/New_York" }; +const cases: { name: string; current: TimeWindowParams; comparison: TimeWindowParams | null; bucket: TimeBucket }[] = [ + { + name: "adjacent rolling windows and identity changes", + current: rolling, + comparison: { ...rolling, past_minutes_start: 2880, past_minutes_end: 1440 }, + bucket: "hour", + }, + { + name: "overlapping periods", + current: rolling, + comparison: { ...rolling, past_minutes_start: 2880, past_minutes_end: 720 }, + bucket: "hour", + }, + { name: "identical periods", current: rolling, comparison: rolling, bucket: "hour" }, + { name: "disabled comparison", current: rolling, comparison: null, bucket: "hour" }, + { name: "DST spring forward", current: day("2024-03-10"), comparison: null, bucket: "hour" }, + { name: "DST fall back", current: day("2024-11-03"), comparison: null, bucket: "hour" }, + { name: "half-hour timezone", current: day("2026-09-19", "Asia/Kolkata"), comparison: null, bucket: "hour" }, + { name: "no events", current: day("2026-09-16"), comparison: null, bucket: "hour" }, + { + name: "daily buckets", + current: { ...day("2026-09-19"), start_date: "2026-09-18" }, + comparison: null, + bucket: "day", + }, + { + name: "exact minute window", + current: { start_datetime: "2026-09-19 21:05:00Z", end_datetime: "2026-09-19 22:01:00Z", time_zone: "UTC" }, + comparison: null, + bucket: "minute", + }, + { name: "all-time hourly activity", current: {}, comparison: null, bucket: "hour" }, + { name: "all-time monthly activity", current: {}, comparison: null, bucket: "month" }, +]; + +describe.skipIf(!process.env.CLICKHOUSE_TEST_URL)("Site cards: ClickHouse results against standard endpoints", () => { + let client: ReturnType; + + beforeAll(() => { + vi.spyOn(Date, "now").mockReturnValue(now); + client = createClient({ + url: process.env.CLICKHOUSE_TEST_URL, + username: process.env.CLICKHOUSE_TEST_USER || "default", + password: process.env.CLICKHOUSE_TEST_PASSWORD || "", + compression: { request: false, response: false }, + clickhouse_settings: { readonly: "2", max_threads: 2, max_execution_time: 10 }, + }); + }); + afterAll(async () => { + vi.restoreAllMocks(); + await client.close(); + }); + + async function run(query: string, params: Record) { + // The standard chart already starts with WITH; put the fixture in that + // clause. Both versions then execute against the identical events CTE. + const sql = /^\s*WITH\b/.test(query) ? query.replace(/^\s*WITH\b/, `${fixture},`) : `${fixture} ${query}`; + const result = await client.query({ query: sql, query_params: params, format: "JSONEachRow" }); + return result.json(); + } + + it.each(cases)( + "$name", + async ({ current, comparison, bucket, name }) => { + const queries = buildSiteCardsQueries({ siteIds, current, comparison, bucket }, now); + const totals = await run>(queries.totals.query, queries.totals.params!); + const series = await run<{ time: string; site_sessions: [number, number | string][] }>( + queries.series.query, + queries.series.params! + ); + + for (const siteId of siteIds) { + const total = totals.find(row => row.site_id === siteId); + for (const [period, window] of [ + ["current", current], + ["previous", comparison], + ] as const) { + if (window === null) { + expect(Number(total?.previous_sessions ?? 0)).toBe(0); + expect(Number(total?.previous_users ?? 0)).toBe(0); + continue; + } + const original = await run<{ sessions: string; users: string }>( + buildOverviewQuery({ timeStatement: resolveTimeWindow(window, now).where() }), + { siteId } + ); + expect(Number(total?.[`${period}_sessions`] ?? 0)).toBe(Number(original[0].sessions)); + expect(Number(total?.[`${period}_users`] ?? 0)).toBe(Number(original[0].users)); + } + + const original = await run<{ time: string; sessions: string }>( + buildOverviewBucketedQuery( + { start_date: "", end_date: "", time_zone: "UTC", ...current, bucket, filters: "" }, + siteId + ), + { siteId } + ); + const actual = series.flatMap(row => { + const counts = new Map(row.site_sessions); + return queries.fillMissingBuckets || counts.has(siteId) + ? [{ time: row.time, sessions: Number(counts.get(siteId) ?? 0) }] + : []; + }); + if (!queries.fillMissingBuckets) { + // The old FULL JOIN selects the left time, so event-only buckets are + // incorrectly labeled 1970. Keep its nonzero session counts, but + // check all unbounded bucket times directly against the fixture. + expect(actual.filter(row => row.sessions > 0)).toEqual( + original + .filter(row => Number(row.sessions) > 0) + .map(row => ({ time: row.time, sessions: Number(row.sessions) })) + ); + const sessions = new Map(); + const bucketOf = (timestamp: string) => + bucket === "hour" ? `${timestamp.slice(0, 13)}:00:00` : `${timestamp.slice(0, 7)}-01 00:00:00`; + const activity = new Map(); + for (const [site, timestamp, session] of events) { + if (site !== siteId) continue; + const time = String(timestamp); + const start = sessions.get(String(session)); + if (!start || time < start) sessions.set(String(session), time); + activity.set(bucketOf(time), 0); + } + for (const start of sessions.values()) { + const time = bucketOf(start); + activity.set(time, activity.get(time)! + 1); + } + expect(actual).toEqual( + [...activity].sort(([a], [b]) => a.localeCompare(b)).map(([time, sessions]) => ({ time, sessions })) + ); + } else { + expect(actual).toEqual(original.map(row => ({ time: row.time, sessions: Number(row.sessions) }))); + } + } + + if (name === "adjacent rolling windows and identity changes") { + const site = totals.find(row => row.site_id === 1)!; + expect(Number(site.current_sessions)).toBe(4); + expect(Number(site.current_users)).toBe(3); + expect(Number(site.previous_sessions)).toBe(2); + expect(Number(site.previous_users)).toBe(1); + } + }, + 30_000 + ); +}); diff --git a/server/src/api/analytics/siteCardsQuery.test.ts b/server/src/api/analytics/siteCardsQuery.test.ts new file mode 100644 index 000000000..a9db68e42 --- /dev/null +++ b/server/src/api/analytics/siteCardsQuery.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildSiteCardsQueries } from "./siteCardsQuery.js"; +import { EFFECTIVE_SESSION_USER_ID } from "./utils/effectiveUserId.js"; + +const current = { past_minutes_start: 1440, past_minutes_end: 0, time_zone: "America/New_York" }; +const comparison = { ...current, past_minutes_start: 2880, past_minutes_end: 1440 }; +const now = Date.parse("2026-09-20T20:41:33Z"); + +describe("raw-event Site card queries", () => { + it("scans events once per query for all requested sites, without materialized views", () => { + const queries = buildSiteCardsQueries({ siteIds: [1, 2, 3], current, comparison, bucket: "hour" }, now); + for (const spec of [queries.totals, queries.series]) { + expect(spec.query.match(/FROM events/g)).toHaveLength(1); + expect(spec.query).not.toMatch(/_mv_target|JOIN\s*\(/); + expect(spec.params).toEqual({ siteIds: [1, 2, 3] }); + expect(spec.query).toContain("site_id IN {siteIds:Array(Int32)}"); + } + }); + + it("resolves identity per site, session AND period using the standard metric definition", () => { + const { totals } = buildSiteCardsQueries({ siteIds: [1, 2], current, comparison, bucket: "hour" }, now); + expect(totals.query).toContain(EFFECTIVE_SESSION_USER_ID); + expect(totals.query).toContain("GROUP BY site_id, session_id, period"); + expect(totals.query).toContain("COUNT(DISTINCT if(period = 0, effective_user_id, NULL))"); + expect(totals.query).toContain("COUNT(DISTINCT if(period = 1, effective_user_id, NULL))"); + }); + + it("allows an event to participate in both overlapping periods", () => { + const { totals } = buildSiteCardsQueries({ siteIds: [1], current, comparison: current, bucket: "hour" }, now); + expect(totals.query).toContain("[0, 1]"); + expect(totals.query).toContain("ARRAY JOIN"); + }); + + it("does not scan a previous window when comparison is disabled", () => { + const { totals } = buildSiteCardsQueries({ siteIds: [1], current, comparison: null, bucket: "hour" }, now); + expect(totals.query).toContain("ARRAY JOIN [0] AS period"); + expect(totals.query).not.toContain("2026-09-18"); + }); + + it("resolves relative bounds from a single clock across both queries", () => { + const clock = vi.spyOn(Date, "now").mockReturnValue(now); + try { + const { totals, series } = buildSiteCardsQueries({ siteIds: [1], current, comparison, bucket: "hour" }); + expect(clock).toHaveBeenCalledTimes(1); + expect(totals.query).toContain("timestamp <= toDateTime('2026-09-20 20:41:33', 'UTC')"); + expect(series.query).toContain("timestamp <= toDateTime('2026-09-20 20:41:33', 'UTC')"); + } finally { + clock.mockRestore(); + } + }); + + it("retains event-only buckets for all-time charts without scanning events twice", () => { + const { series, fillMissingBuckets } = buildSiteCardsQueries( + { siteIds: [1], current: {}, comparison: null, bucket: "hour" }, + now + ); + expect(fillMissingBuckets).toBe(false); + expect(series.query).toContain("groupUniqArray"); + expect(series.query).toContain("countIf(active_bucket ="); + expect(series.query).not.toContain("WITH FILL"); + expect(series.query.match(/FROM events/g)).toHaveLength(1); + }); + + it("supports exact windows and sub-hour buckets without rounding the request", () => { + const { series } = buildSiteCardsQueries( + { + siteIds: [1], + comparison: null, + bucket: "five_minutes", + current: { + start_datetime: "2026-09-18 10:32:00Z", + end_datetime: "2026-09-18 11:17:00Z", + time_zone: "Asia/Kolkata", + }, + }, + now + ); + expect(series.query).toContain("toStartOfFiveMinutes"); + expect(series.query).toContain("timestamp >= toDateTime('2026-09-18 10:32:00', 'UTC')"); + expect(series.query).toContain("timestamp < toDateTime('2026-09-18 11:17:00', 'UTC')"); + }); +}); diff --git a/server/src/api/analytics/siteCardsQuery.ts b/server/src/api/analytics/siteCardsQuery.ts new file mode 100644 index 000000000..de4d00365 --- /dev/null +++ b/server/src/api/analytics/siteCardsQuery.ts @@ -0,0 +1,86 @@ +import { TimeBucket } from "@rybbit/shared"; +import { QuerySpec } from "./utils/analyticsQuery.js"; +import { EFFECTIVE_SESSION_USER_ID } from "./utils/effectiveUserId.js"; +import { resolveTimeWindow, TimeWindowParams } from "./utils/timeWindow.js"; + +export interface SiteCardQueryParams { + siteIds: number[]; + current: TimeWindowParams; + comparison: TimeWindowParams | null; + bucket: TimeBucket; +} + +export interface SiteCardQueries { + totals: QuerySpec; + series: QuerySpec; + fillMissingBuckets: boolean; +} + +export function buildSiteCardsQueries( + { siteIds, current, comparison, bucket }: SiteCardQueryParams, + now = Date.now() +): SiteCardQueries { + const window = resolveTimeWindow(current, now); + const previous = comparison === null ? null : resolveTimeWindow(comparison, now); + const currentPredicate = `(1 ${window.where()})`; + const previousPredicate = previous ? `(1 ${previous.where()})` : "0"; + // An event in overlapping windows belongs to both periods. Resolve identity + // AFTER this split, using the same session-level definition as getOverview. + // Otherwise identify() in the current period could rewrite previous users. + const periods = previous ? `if(${currentPredicate}, if(${previousPredicate}, [0, 1], [0]), [1])` : "[0]"; + const params = { siteIds }; + + const totals: QuerySpec = { + query: ` + SELECT site_id, + countIf(period = 0) AS current_sessions, + COUNT(DISTINCT if(period = 0, effective_user_id, NULL)) AS current_users, + countIf(period = 1) AS previous_sessions, + COUNT(DISTINCT if(period = 1, effective_user_id, NULL)) AS previous_users + FROM ( + SELECT site_id, session_id, period, + ${EFFECTIVE_SESSION_USER_ID} AS effective_user_id + FROM events + ARRAY JOIN ${periods} AS period + WHERE site_id IN {siteIds:Array(Int32)} + AND (${currentPredicate} OR ${previousPredicate}) + GROUP BY site_id, session_id, period + ) + GROUP BY site_id + `, + params, + }; + + // Bounded charts get empty buckets from WITH FILL. For all-time, retain the + // old chart's event-only buckets (zero new sessions) without a second events + // scan: collect each session's distinct active buckets alongside its start. + const sessionBucket = window.bucketed("session_start", bucket); + const counts = ` + SELECT site_id, + ${window.isAllTime ? "active_bucket" : sessionBucket} AS time, + ${window.isAllTime ? `countIf(active_bucket = ${sessionBucket})` : "count()"} AS sessions + FROM ( + SELECT site_id, session_id, min(timestamp) AS session_start + ${window.isAllTime ? `, groupUniqArray(${window.bucketed("timestamp", bucket)}) AS active_buckets` : ""} + FROM events + WHERE site_id IN {siteIds:Array(Int32)} ${window.where()} + GROUP BY site_id, session_id + ) + ${window.isAllTime ? "ARRAY JOIN active_buckets AS active_bucket" : ""} + GROUP BY site_id, time + `; + + return { + totals, + fillMissingBuckets: !window.isAllTime, + series: { + query: ` + SELECT time, groupArray((site_id, sessions)) AS site_sessions + FROM (${counts}) + GROUP BY time + ORDER BY time ${window.fill(bucket)} + `, + params, + }, + }; +} diff --git a/server/src/api/analytics/utils/timeWindow.ts b/server/src/api/analytics/utils/timeWindow.ts index 15c06b8cc..1c95b1013 100644 --- a/server/src/api/analytics/utils/timeWindow.ts +++ b/server/src/api/analytics/utils/timeWindow.ts @@ -162,7 +162,7 @@ type ResolvedWindow = /** Formats an instant the way ClickHouse's `toDateTime` wants it. */ const toClickhouseInstant = (date: Date) => date.toISOString().slice(0, 19).replace("T", " "); -function resolve(params: TimeWindowParams): ResolvedWindow { +function resolve(params: TimeWindowParams, now: number): ResolvedWindow { const timeZone = params.time_zone || "UTC"; // An unusable timezone makes every window meaningless — a date range means @@ -201,7 +201,6 @@ function resolve(params: TimeWindowParams): ResolvedWindow { // Resolved to absolute instants here, once, so the predicate and the fill // describe the same window: `now()` used to be read separately by each // builder — and four times over inside the overview query alone. - const now = Date.now(); return { kind: "pastMinutes", start: toClickhouseInstant(new Date(now - parsed.data.start * 60 * 1000)), @@ -356,8 +355,8 @@ function fillClause(window: ResolvedWindow, bucket: TimeBucket): string { * the endpoints have always applied. A mode whose params are incomplete or * malformed is skipped rather than failing the whole resolution. */ -export function resolveTimeWindow(params: TimeWindowParams): TimeWindow { - const window = resolve(params); +export function resolveTimeWindow(params: TimeWindowParams, now = Date.now()): TimeWindow { + const window = resolve(params, now); // An all-time window still buckets — it just doesn't fill — so the display // timezone has to survive a window that resolved to no bounds. const timeZone = window.kind === "all" ? params.time_zone || "UTC" : window.timeZone; diff --git a/server/src/index.ts b/server/src/index.ts index 806f8ca80..a2f6ea002 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -69,6 +69,8 @@ import { getOverviewBucketed, getOverviewBucketedLite, getOverviewLite, + getSiteCardsLite, + getSiteCards, getPageTitles, getPerformanceByDimension, getPerformanceOverview, @@ -570,6 +572,8 @@ async function organizationsRoutes(fastify: FastifyInstance) { // Organizations fastify.get("/organizations", getMyOrganizations); fastify.get("/organizations/:organizationId/sites", orgOrgRead, getSitesFromOrg); + fastify.post("/organizations/:organizationId/site-cards-lite", orgAnalyticsRead, getSiteCardsLite); + fastify.post("/organizations/:organizationId/site-cards", orgAnalyticsRead, getSiteCards); fastify.post("/organizations/:organizationId/sites", orgAdminSitesWrite, addSite); // Landing-page domain input: creates an owner-less site reachable only by // its private link key. Public, so cap creations per IP.