From 4fd8f7bf3decb85f811034d7a87398856149353c Mon Sep 17 00:00:00 2001 From: zetazzz Date: Mon, 21 Sep 2026 10:47:15 +0000 Subject: [PATCH] Move scoped introspection benchmark to TypeScript and keep results local --- packages/perf-harness/.gitignore | 3 + packages/perf-harness/README.md | 5 +- .../__tests__/entrypoints.test.ts | 49 ++++- .../scoped-introspection-environment.json | 76 -------- .../scoped-introspection-samples.csv | 43 ----- .../benchmarks/scoped-introspection.cjs | 75 -------- .../benchmarks/scoped-introspection.md | 14 +- packages/perf-harness/package.json | 7 +- .../benchmarks/scoped-introspection/run.ts | 167 ++++++++++++++++++ pnpm-lock.yaml | 6 + 10 files changed, 240 insertions(+), 205 deletions(-) delete mode 100644 packages/perf-harness/benchmarks/scoped-introspection-environment.json delete mode 100644 packages/perf-harness/benchmarks/scoped-introspection-samples.csv delete mode 100644 packages/perf-harness/benchmarks/scoped-introspection.cjs create mode 100644 packages/perf-harness/src/benchmarks/scoped-introspection/run.ts diff --git a/packages/perf-harness/.gitignore b/packages/perf-harness/.gitignore index 064721367..b2472d197 100644 --- a/packages/perf-harness/.gitignore +++ b/packages/perf-harness/.gitignore @@ -3,3 +3,6 @@ /cache-micro-report.json /cache-postgres-report.json /cache-variants-report.json + +# Local outputs from the scoped introspection benchmark +/benchmarks/scoped-introspection/results/ diff --git a/packages/perf-harness/README.md b/packages/perf-harness/README.md index ff4044b91..5c6451bd4 100644 --- a/packages/perf-harness/README.md +++ b/packages/perf-harness/README.md @@ -104,8 +104,9 @@ versions and catalog sizes; runtime validation and process startup are outside `buildMs`. `processPeakRss` is the worker peak measured after runtime validation, before service release. -The [default scoped comparison](benchmarks/scoped-introspection.md) includes a -reproduction command, measured results, and the individual timing/memory samples. +The [default scoped comparison](benchmarks/scoped-introspection.md) records the +historical method, summarized results, limitations, and a reproduction command +for the typed runner. Generated reports and per-sample results stay local. ## Cache capacity experiments diff --git a/packages/perf-harness/__tests__/entrypoints.test.ts b/packages/perf-harness/__tests__/entrypoints.test.ts index bd6f13a7f..db0470f79 100644 --- a/packages/perf-harness/__tests__/entrypoints.test.ts +++ b/packages/perf-harness/__tests__/entrypoints.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from 'node:child_process'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -31,6 +31,8 @@ describe('built package entry points', () => { '--eval', `const assert = require('node:assert/strict'); const library = require(${JSON.stringify(entry)}); + const scoped = require(${JSON.stringify(resolve(artifactRoot, 'benchmarks/scoped-introspection/run.js'))}); + assert.equal(typeof scoped.main, 'function'); assert.equal(typeof library.runBenchmarkSuite, 'function'); assert.equal(typeof library.prepareFixture, 'function');`, ]); @@ -51,6 +53,8 @@ describe('built package entry points', () => { ], `import assert from 'node:assert/strict'; import * as library from ${JSON.stringify(entry)}; + import * as scoped from ${JSON.stringify(pathToFileURL(resolve(artifactRoot, 'esm/benchmarks/scoped-introspection/run.js')).href)}; + assert.equal(typeof scoped.main, 'function'); assert.equal(typeof require, 'undefined'); assert.equal(typeof module, 'undefined'); assert.equal(typeof library.runBenchmarkSuite, 'function'); @@ -113,4 +117,47 @@ describe('built package entry points', () => { expect(result.stdout).toBe(''); expect(result.stderr).toContain('--database-url is required'); }); + + test('shows scoped introspection help without opening a database', () => { + const result = node([ + resolve(artifactRoot, 'benchmarks/scoped-introspection/run.js'), + '--help', + ]); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stdout).toContain('Usage: scoped:introspection [--output DIRECTORY]'); + expect(result.stderr).toBe(''); + }); + + test.each([ + [['--output'], 'expected --name value'], + [['--output', ''], '--output must be a non-empty directory'], + [['--unknown', 'value'], "unsupported argument '--unknown'"], + ])('rejects invalid scoped introspection arguments %j', (args, message) => { + const result = node([ + resolve(artifactRoot, 'benchmarks/scoped-introspection/run.js'), + ...args, + ]); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr).toContain(message); + }); + + test('rejects a file as the scoped output directory before opening a database', async () => { + const directory = await mkdtemp(resolve(tmpdir(), 'cperf-scoped-entry-')); + const output = resolve(directory, 'existing-file'); + try { + await writeFile(output, 'preserve me'); + const result = node([ + resolve(artifactRoot, 'benchmarks/scoped-introspection/run.js'), + '--output', output, + ]); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr).toContain('EEXIST'); + expect(await readFile(output, 'utf8')).toBe('preserve me'); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); }); diff --git a/packages/perf-harness/benchmarks/scoped-introspection-environment.json b/packages/perf-harness/benchmarks/scoped-introspection-environment.json deleted file mode 100644 index 5741b80b1..000000000 --- a/packages/perf-harness/benchmarks/scoped-introspection-environment.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "capturedAt": "2026-09-10T03:08:22.678Z", - "node": "v24.20.0", - "platform": "linux", - "architecture": "x64", - "cpu": "Intel(R) Xeon(R) Gold 6152 CPU @ 2.10GHz", - "logicalCpus": 3, - "totalMemory": 4105740288, - "loadAverageBefore": [ - 6.24, - 5.67, - 5.14 - ], - "adaptorSessionSettings": [ - { - "name": "jit", - "setting": "on", - "unit": null - }, - { - "name": "jit_above_cost", - "setting": "100000", - "unit": null - }, - { - "name": "jit_inline_above_cost", - "setting": "500000", - "unit": null - }, - { - "name": "jit_optimize_above_cost", - "setting": "-1", - "unit": null - }, - { - "name": "max_parallel_workers_per_gather", - "setting": "2", - "unit": null - }, - { - "name": "statement_timeout", - "setting": "0", - "unit": "ms" - }, - { - "name": "work_mem", - "setting": "4096", - "unit": "kB" - } - ], - "postgresVersion": "18.6", - "mainBase": "e008e936", - "crystalSource": "441cef73b0a12ed24343b7f38241af28b6454280", - "loadAverageAfter": { - "0": [ - 5.57, - 5.6, - 5.16 - ], - "10": [ - 4.82, - 5.37, - 5.12 - ], - "50": [ - 5.25, - 5.3, - 5.12 - ] - }, - "compiledCodeSha256": { - "graphile/graphile-scoped-introspection/dist/plugin.js": "68bb93b9e373b916b5453d026f92fc9f239ac4a29c5fe4cbde05e4b57324fa0f", - "graphile/graphile-scoped-introspection/dist/scoped-introspection-query.js": "82f61359eec84444bd2d425274740385364a7de258ba7896d62114dbf48d81fe", - "packages/perf-harness/dist/scoped-introspection-worker.js": "929a29ab9d3b061867a976d45a35b59da9e5183923457c64ee74fb5551a9b999" - } -} diff --git a/packages/perf-harness/benchmarks/scoped-introspection-samples.csv b/packages/perf-harness/benchmarks/scoped-introspection-samples.csv deleted file mode 100644 index 6916a3862..000000000 --- a/packages/perf-harness/benchmarks/scoped-introspection-samples.csv +++ /dev/null @@ -1,43 +0,0 @@ -unrelated_schemas,case,repetition,pid,build_ms,retained_heap_bytes,peak_rss_bytes,schema_sha256,validation_passed -0,stock,1,1314792,1117.40321,27647640,124674048,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,1,1314841,4339.743367,27753216,124731392,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,2,1314899,4495.784806000001,27752816,124575744,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,2,1315037,1214.189011,27647016,124628992,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,3,1315054,1204.0237589999997,27647000,124841984,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,3,1315088,4166.057228,27751072,124690432,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,4,1315153,1322.88216,27646232,124727296,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,4,1315179,4199.1300599999995,27752768,124866560,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,5,1315291,4433.801736,27752200,124694528,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,5,1315404,1195.751804,27648016,124907520,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,6,1315442,1254.9538679999998,27648208,124628992,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,6,1315459,3745.3831609999997,27749712,125816832,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,scoped,7,1315519,4056.742956,27752600,125448192,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -0,stock,7,1315571,1120.6722579999998,27647560,125673472,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,1,1315792,2543.90956,43804168,195346432,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,1,1315875,4541.515913,27750280,125022208,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,2,1315949,3940.2051029999993,27752320,124772352,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,2,1315989,2275.408082,43825144,174792704,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,3,1316054,2206.7710420000003,43828808,180027392,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,3,1316096,4009.055742,27748664,124977152,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,4,1316182,2393.0753669999995,43824720,180187136,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,4,1316251,4023.3548249999994,27750816,125210624,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,5,1316294,4580.057837,27754144,125186048,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,5,1316338,2394.544593,43800568,196788224,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,6,1316418,2630.196468,43824760,196124672,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,6,1316436,3987.7613659999997,27753080,125018112,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,scoped,7,1316499,4745.523598,27752296,125063168,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -10,stock,7,1316628,2593.1881860000003,43823160,195518464,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,1,1317266,7757.512533,109930952,303149056,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,1,1317382,4675.1935969999995,27752704,126042112,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,2,1317455,4206.750878,27754672,125587456,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,2,1317536,6911.457863,108211872,341778432,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,3,1317627,6630.055281999999,108211632,332468224,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,3,1317716,4127.204637,27752832,125562880,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,4,1317739,6539.152844,108198952,337833984,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,4,1317880,4525.434472999999,27751136,126095360,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,5,1317957,4266.888,27752680,125861888,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,5,1318017,6457.588834,109907680,305082368,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,6,1318108,7062.589355000001,109909088,307884032,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,6,1318173,4152.237724,27753064,126033920,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,scoped,7,1318247,4548.305887,27754896,125759488,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True -50,stock,7,1318327,6383.482334,108250840,328863744,3b9dab5243611a57da1c0721ea3bc1aba455051124e43549e3c326178b3e97c1,True diff --git a/packages/perf-harness/benchmarks/scoped-introspection.cjs b/packages/perf-harness/benchmarks/scoped-introspection.cjs deleted file mode 100644 index b607771a9..000000000 --- a/packages/perf-harness/benchmarks/scoped-introspection.cjs +++ /dev/null @@ -1,75 +0,0 @@ -// Reproduce the default stock/scoped catalog-scaling comparison. -// Run after building perf-harness, graphile-scoped-introspection and pgsql-test. -const fs = require('node:fs'); -const path = require('node:path'); -const os = require('node:os'); -const root = process.argv[2]; -const output = process.argv[3]; -if (!root || !output) throw new Error('usage: node scoped-introspection.cjs ABSOLUTE_WORKTREE OUTPUT'); -const { createRequire } = require('node:module'); -const req = createRequire(path.join(root, 'packages/perf-harness/package.json')); -const { getConnections } = require(path.join(root, 'postgres/pgsql-test/dist')); -const { buildConnectionString } = require(path.join(root, 'postgres/pg-cache/dist')); -const { prepareFixture, makeScopedIntrospectionSuite, runBenchmarkSuite } = require(path.join(root, 'packages/perf-harness/dist')); -(async () => { - fs.mkdirSync(output, { recursive: true }); - const conn = await getConnections({}, []); - try { - const c = conn.pg.config; - const databaseUrl = buildConnectionString(c.user, c.password, c.host, c.port, c.database); - const fixture = await prepareFixture({ databaseUrl, schema: 'cperf_target', tables: 8 }); - await conn.pg.query(`insert into cperf_target.account (external_id, name) values ('11111111-1111-1111-1111-111111111111', 'benchmark account'); - insert into cperf_target.entity_1 (account_id, title) values (1, 'benchmark entity');`); - const suite = makeScopedIntrospectionSuite({ - schemas: ['cperf_target'], - runtimeCheck: { - query: '{ allAccounts { nodes { name } } allEntity1S { nodes { title accountByAccountId { name } } } entity1ByAccount(requestedAccountId: "1") { nodes { title } } }', - expectedData: { allAccounts: { nodes: [{ name: 'benchmark account' }] }, allEntity1S: { nodes: [{ title: 'benchmark entity', accountByAccountId: { name: 'benchmark account' } }] }, entity1ByAccount: { nodes: [{ title: 'benchmark entity' }] } }, - }, - }); - const worker = path.join(root, 'packages/perf-harness/dist/scoped-introspection-worker.js'); - const environment = { - capturedAt: new Date().toISOString(), - node: process.version, platform: process.platform, architecture: process.arch, - cpu: os.cpus()[0].model, logicalCpus: os.cpus().length, totalMemory: os.totalmem(), - fixture, loadAverageBefore: os.loadavg(), - postgresSettings: (await conn.pg.query("select name, setting, unit from pg_settings where name in ('jit','jit_above_cost','work_mem','statement_timeout','max_parallel_workers_per_gather','shared_buffers') order by name")).rows, - method: 'Fresh Node process per sample, default PostGraphile adaptor session settings for both cases, database caches warmed by discarded pair at each scale, seeded interleaving, no historical implementation or catalogTypes override.', - }; - fs.writeFileSync(path.join(output, 'environment.json'), JSON.stringify(environment, null, 2)); - let previous = 0; - for (const schemas of [0, 10, 50]) { - for (let i = previous; i < schemas; i++) await prepareFixture({ databaseUrl, schema: `cperf_noise_${i}`, tables: 20 }); - previous = schemas; - const counts = await conn.pg.query(`select (select count(*)::int from pg_namespace where nspname like 'cperf_%') as schemas, (select count(*)::int from pg_class c join pg_namespace n on n.oid=c.relnamespace where n.nspname like 'cperf_%' and c.relkind='r') as tables, (select count(*)::int from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname like 'cperf_%') as functions`); - const options = { databaseUrl, repetitions: 1, seed: 20260910, order: null, workerTimeoutMs: 300000 }; - const warm = await runBenchmarkSuite(suite, options, worker); - fs.writeFileSync(path.join(output, `warmup-${schemas}.json`), JSON.stringify(warm, null, 2)); - if (!warm.validation.allRunsSucceeded || !warm.validation.schemaGroupsEquivalent) throw new Error(JSON.stringify(warm.validation)); - const report = await runBenchmarkSuite(suite, { ...options, repetitions: 7 }, worker); - report.catalog = counts.rows[0]; - report.loadAverageAfter = os.loadavg(); - fs.writeFileSync(path.join(output, `report-${schemas}.json`), JSON.stringify(report, null, 2)); - if (!report.validation.allRunsSucceeded || !report.validation.schemaGroupsEquivalent) throw new Error(JSON.stringify(report.validation)); - console.log(JSON.stringify({ unrelatedSchemas: schemas, catalog: counts.rows[0], validation: report.validation, medians: Object.fromEntries(Object.entries(report.summaries).map(([k,v]) => [k, { buildMs: v.buildMs.median, heapMiB: v.heapUsedAfterBuild.median/1048576, peakRssMiB: v.processPeakRss.median/1048576 }])) })); - } - // SQL-only diagnostics after measured samples, through the same default - // PostGraphile adaptor as the workers (including its built-in JIT setting). - const stockQuery = req('postgraphile/graphile-build-pg/pg-introspection').makeIntrospectionQuery(); - const scopedQuery = req('graphile-scoped-introspection').makeSchemaScopedIntrospectionPlan(['cperf_target']).query; - const { makePgService } = req('postgraphile/adaptors/pg'); - const { withPgClientFromPgService } = req('postgraphile/@dataplan/pg'); - const service = makePgService({ name: 'main', connectionString: databaseUrl, schemas: ['cperf_target'], pubsub: false }); - try { - await withPgClientFromPgService(service, null, async client => { - environment.adaptorSessionSettings = (await client.query({ text: "select name, setting, unit from pg_settings where name in ('jit','jit_above_cost','jit_inline_above_cost','jit_optimize_above_cost','work_mem','statement_timeout','max_parallel_workers_per_gather') order by name" })).rows; - fs.writeFileSync(path.join(output, 'environment.json'), JSON.stringify(environment, null, 2)); - const plans = {}; - for (const [name, query] of [['stock', { text: stockQuery, values: [] }], ['scoped', scopedQuery]]) { - plans[name] = (await client.query({ text: 'explain (analyze, buffers, format json) ' + query.text, values: query.values })).rows[0]['QUERY PLAN']; - } - fs.writeFileSync(path.join(output, 'plans-default.json'), JSON.stringify(plans, null, 2)); - }); - } finally { await service.release(); } - } finally { await conn.teardown(); } -})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/packages/perf-harness/benchmarks/scoped-introspection.md b/packages/perf-harness/benchmarks/scoped-introspection.md index d198e2b3a..93bd6b76f 100644 --- a/packages/perf-harness/benchmarks/scoped-introspection.md +++ b/packages/perf-harness/benchmarks/scoped-introspection.md @@ -2,6 +2,8 @@ Schema-scoped introspection reduces catalog work for a small API within a large database. This experiment compares stock with the copied CNC plugin enabled by `gather.pgScopedIntrospection.main: true` on the main-based implementation. +This is a historical run captured on 2026-09-10. Its measurements describe that implementation and environment, not the current runner build. + ## Method - The target schema stays fixed at 9 tables and 8 functions. Each unrelated schema adds 21 tables and 20 functions. @@ -11,7 +13,7 @@ Schema-scoped introspection reduces catalog work for a small API within a large - Every sample must pass actual table, relation, and function queries and schema hash equivalence. Retained heap is measured after validation and forced GC; peak RSS is read at that point, before service release. - This is a shared host with background load and a synthetic fixture. Medians and ranges describe these samples and do not establish a production speedup. -Environment: Node v24.20.0, PostgreSQL 18.6, 3 logical CPUs, 3.82 GiB RAM. Full parameters are in [environment metadata](scoped-introspection-environment.json). +Environment: Linux x64 on an Intel Xeon Gold 6152 at 2.10 GHz, Node v24.20.0, PostgreSQL 18.6, 3 logical CPUs, and 3.82 GiB RAM. The initial 1/5/15-minute load averages were 6.24/5.67/5.14. The run used `jit=on`, `jit_above_cost=100000`, `jit_inline_above_cost=500000`, `jit_optimize_above_cost=-1`, `max_parallel_workers_per_gather=2`, `statement_timeout=0`, and `work_mem=4096 kB`. The captured Constructive main base was `e008e936`; the copied query source was Crystal #2 commit `441cef73b0a12ed24343b7f38241af28b6454280`. Historical [environment metadata](https://github.com/constructive-io/constructive/blob/66c6bf8b7892a0a19169ddf9ae36b157707abef5/packages/perf-harness/benchmarks/scoped-introspection-environment.json) records the compiled-code hashes and per-scale load averages. ## Results @@ -21,7 +23,7 @@ Environment: Node v24.20.0, PostgreSQL 18.6, 3 logical CPUs, 3.82 GiB RAM. Full | 10 / 210 | 2.395 s | 4.023 s | +68.0% | 41.8 / 26.5 MiB | 186.3 / 119.2 MiB | | 50 / 1050 | 6.630 s | 4.267 s | -35.6% | 103.2 / 26.5 MiB | 313.6 / 120.0 MiB | -All 42 measured samples passed validation and used distinct PIDs. Individual measurements are in [samples](scoped-introspection-samples.csv). +All 42 measured samples passed validation and used distinct PIDs. The original [sample CSV](https://github.com/constructive-io/constructive/blob/66c6bf8b7892a0a19169ddf9ae36b157707abef5/packages/perf-harness/benchmarks/scoped-introspection-samples.csv) remains available at the commit that recorded this run. | Unrelated schemas | Stock build range | Scoped build range | |---|---:|---:| @@ -46,10 +48,10 @@ Use the repository development instructions to load a test-admin PostgreSQL conn ```sh pnpm install --frozen-lockfile -pnpm -r --filter '@constructive-io/perf-harness...' --filter 'pgsql-test...' run build -node packages/perf-harness/benchmarks/scoped-introspection.cjs "$PWD" /tmp/cnc-scoped-results +pnpm -r --filter '@constructive-io/perf-harness...' run build +pnpm --filter @constructive-io/perf-harness scoped:introspection ``` -The output directory includes full reports, discarded warm-ups, database/session metadata, and SQL plans. The committed CSV and JSON contain only selected non-secret measurements and environment data. +By default, outputs are written under `packages/perf-harness/benchmarks/scoped-introspection/results/`; append `--output /tmp/cnc-scoped-results` to the runner command to choose another directory. The output contains `environment.json`, `warmup-{0,10,50}.json`, `report-{0,10,50}.json`, and `plans-default.json` SQL diagnostics. The default results directory is ignored by Git. -The copied query comes from [Crystal #2](https://github.com/constructive-io/crystal/pull/2), commit `441cef73b0a12ed24343b7f38241af28b6454280`. These measurements were taken after integration onto Constructive main `e008e936`; compiled-code hashes are included in the environment metadata. +The copied query comes from [Crystal #2](https://github.com/constructive-io/crystal/pull/2), commit `441cef73b0a12ed24343b7f38241af28b6454280`. The original result artifacts are linked above for historical reference; the current runner reproduces the methodology, not the 2026-09-10 measurements. diff --git a/packages/perf-harness/package.json b/packages/perf-harness/package.json index 546145411..0230425f5 100644 --- a/packages/perf-harness/package.json +++ b/packages/perf-harness/package.json @@ -19,7 +19,8 @@ "cache:micro": "node dist/benchmarks/grafast-cache/micro.js", "cache:postgres": "node dist/benchmarks/grafast-cache/postgres.js", "cache:variants": "node dist/benchmarks/grafast-cache/variants.js", - "cache:analyze": "node dist/benchmarks/grafast-cache/analyze.js" + "cache:analyze": "node dist/benchmarks/grafast-cache/analyze.js", + "scoped:introspection": "node dist/benchmarks/scoped-introspection/run.js" }, "dependencies": { "@dataplan/pg": "1.1.1", @@ -36,7 +37,9 @@ "devDependencies": { "@types/node": "^22.19.11", "@types/pg": "^8.20.4", - "makage": "^0.8.0" + "makage": "^0.8.0", + "pg-cache": "workspace:^", + "pgsql-test": "workspace:^" }, "engines": { "node": ">=22" diff --git a/packages/perf-harness/src/benchmarks/scoped-introspection/run.ts b/packages/perf-harness/src/benchmarks/scoped-introspection/run.ts new file mode 100644 index 000000000..f2c26ea03 --- /dev/null +++ b/packages/perf-harness/src/benchmarks/scoped-introspection/run.ts @@ -0,0 +1,167 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { cpus, loadavg, totalmem } from 'node:os'; +import { join, resolve } from 'node:path'; + +import { makeSchemaScopedIntrospectionPlan } from 'graphile-scoped-introspection'; +import { buildConnectionString } from 'pg-cache'; +import { withPgClientFromPgService } from 'postgraphile/@dataplan/pg'; +import { makePgService } from 'postgraphile/adaptors/pg'; +import { makeIntrospectionQuery } from 'postgraphile/graphile-build-pg/pg-introspection'; + +import { prepareFixture } from '../../fixture'; +import { parseValueArgs } from '../../process'; +import type { RunSuiteOptions } from '../../run'; +import { runBenchmarkSuite } from '../../run'; +import { makeScopedIntrospectionSuite } from '../../scoped-introspection-suite'; + +interface SessionSetting { + name: string; + setting: string; + unit: string | null; +} + +interface CatalogCounts { + schemas: number; + tables: number; + functions: number; +} + +const usage = `Usage: scoped:introspection [--output DIRECTORY] + +Compare stock and default scoped introspection using the standard PG* environment. +Results default to benchmarks/scoped-introspection/results in the package directory. +`; + +export async function main(args = process.argv.slice(2)): Promise { + if (args.length === 1 && args[0] === '--help') { + process.stdout.write(usage); + return; + } + const { values } = parseValueArgs(args); + for (const name of values.keys()) { + if (name !== 'output') throw new Error(`unsupported argument '--${name}'`); + } + const output = values.get('output') ?? resolve( + __dirname, '../../../benchmarks/scoped-introspection/results' + ); + if (!output.trim()) throw new Error('--output must be a non-empty directory'); + mkdirSync(output, { recursive: true }); + const writeJson = (name: string, value: unknown): void => { + writeFileSync(join(output, name), JSON.stringify(value, null, 2)); + }; + + // Load the database harness only after validating command-line arguments. + const { getConnections } = await import('pgsql-test'); + const conn = await getConnections({}, []); + try { + const c = conn.pg.config; + const databaseUrl = buildConnectionString(c.user, c.password, c.host, c.port, c.database); + const fixture = await prepareFixture({ databaseUrl, schema: 'cperf_target', tables: 8 }); + await conn.pg.query(`insert into cperf_target.account (external_id, name) values ('11111111-1111-1111-1111-111111111111', 'benchmark account'); + insert into cperf_target.entity_1 (account_id, title) values (1, 'benchmark entity');`); + const suite = makeScopedIntrospectionSuite({ + schemas: ['cperf_target'], + runtimeCheck: { + query: '{ allAccounts { nodes { name } } allEntity1S { nodes { title accountByAccountId { name } } } entity1ByAccount(requestedAccountId: "1") { nodes { title } } }', + expectedData: { + allAccounts: { nodes: [{ name: 'benchmark account' }] }, + allEntity1S: { nodes: [{ title: 'benchmark entity', accountByAccountId: { name: 'benchmark account' } }] }, + entity1ByAccount: { nodes: [{ title: 'benchmark entity' }] }, + }, + }, + }); + const worker = resolve(__dirname, '../../scoped-introspection-worker.js'); + const environment = { + capturedAt: new Date().toISOString(), + node: process.version, + platform: process.platform, + architecture: process.arch, + cpu: cpus()[0].model, + logicalCpus: cpus().length, + totalMemory: totalmem(), + fixture, + loadAverageBefore: loadavg(), + postgresSettings: (await conn.pg.query( + "select name, setting, unit from pg_settings where name in ('jit','jit_above_cost','work_mem','statement_timeout','max_parallel_workers_per_gather','shared_buffers') order by name" + )).rows, + adaptorSessionSettings: [] as readonly SessionSetting[], + method: 'Fresh Node process per sample, default PostGraphile adaptor session settings for both cases, database caches warmed by discarded pair at each scale, seeded interleaving, no historical implementation or catalogTypes override.', + }; + writeJson('environment.json', environment); + let previous = 0; + for (const schemas of [0, 10, 50]) { + for (let i = previous; i < schemas; i++) { + await prepareFixture({ databaseUrl, schema: `cperf_noise_${i}`, tables: 20 }); + } + previous = schemas; + const counts = await conn.pg.query( + "select (select count(*)::int from pg_namespace where nspname like 'cperf_%') as schemas, (select count(*)::int from pg_class c join pg_namespace n on n.oid=c.relnamespace where n.nspname like 'cperf_%' and c.relkind='r') as tables, (select count(*)::int from pg_proc p join pg_namespace n on n.oid=p.pronamespace where n.nspname like 'cperf_%') as functions" + ); + const options: RunSuiteOptions = { databaseUrl, repetitions: 1, seed: 20260910, order: null, workerTimeoutMs: 300000 }; + const warm = await runBenchmarkSuite(suite, options, worker); + writeJson(`warmup-${schemas}.json`, warm); + if (!warm.validation.allRunsSucceeded || !warm.validation.schemaGroupsEquivalent) { + throw new Error(JSON.stringify(warm.validation)); + } + const report = { + ...await runBenchmarkSuite(suite, { ...options, repetitions: 7 }, worker), + catalog: counts.rows[0], + loadAverageAfter: loadavg(), + }; + writeJson(`report-${schemas}.json`, report); + if (!report.validation.allRunsSucceeded || !report.validation.schemaGroupsEquivalent) { + throw new Error(JSON.stringify(report.validation)); + } + console.log(JSON.stringify({ + unrelatedSchemas: schemas, + catalog: counts.rows[0], + validation: report.validation, + medians: Object.fromEntries(Object.entries(report.summaries).map(([name, summary]) => [name, { + buildMs: summary.buildMs.median, + heapMiB: summary.heapUsedAfterBuild.median / 1048576, + peakRssMiB: summary.processPeakRss.median / 1048576, + }])), + })); + } + + // Diagnostics follow all measured samples and use the same default adaptor. + const stockQuery = makeIntrospectionQuery(); + const scopedQuery = makeSchemaScopedIntrospectionPlan(['cperf_target']).query; + const service = makePgService({ name: 'main', connectionString: databaseUrl, schemas: ['cperf_target'], pubsub: false }); + try { + await withPgClientFromPgService(service, null, async (client) => { + environment.adaptorSessionSettings = (await client.query({ + text: "select name, setting, unit from pg_settings where name in ('jit','jit_above_cost','jit_inline_above_cost','jit_optimize_above_cost','work_mem','statement_timeout','max_parallel_workers_per_gather') order by name", + })).rows; + writeJson('environment.json', environment); + const plans: Record = {}; + const queries: Record = { + stock: { text: stockQuery, values: [] }, + scoped: scopedQuery, + }; + for (const [name, query] of Object.entries(queries)) { + plans[name] = (await client.query<{ 'QUERY PLAN': unknown }>({ + text: 'explain (analyze, buffers, format json) ' + query.text, + values: [...query.values], + })).rows[0]['QUERY PLAN']; + } + writeJson('plans-default.json', plans); + }); + } finally { + await service.release(); + } + } finally { + await conn.teardown(); + } +} + +if ( + typeof require !== 'undefined' && + typeof module !== 'undefined' && + require.main === module +) { + void main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08f6ba2b3..78fffc290 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2836,6 +2836,12 @@ importers: makage: specifier: ^0.8.0 version: 0.8.0 + pg-cache: + specifier: workspace:^ + version: link:../../postgres/pg-cache/dist + pgsql-test: + specifier: workspace:^ + version: link:../../postgres/pgsql-test/dist packages/postmaster: dependencies: