diff --git a/CHANGELOG.md b/CHANGELOG.md index 512c107..8665525 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,115 @@ ## Upcoming release -### Performance +### Breaking changes — pluggable database adapter + +The library now talks to a small async `GtfsDatabase` interface. sql.js becomes one adapter among others (better-sqlite3 ships in the box; op-sqlite / expo-sqlite / … pluggable by the user). Three things change at every call site: (1) query methods return `Promise`, (2) an `adapter` is required, (3) `sql.js` is an optional peer dependency — you install it yourself. + +See the full migration write-up in [README](README.md) and the [Usage Guide](documents/guide.md#creating-an-instance). + +#### What's unchanged + +- All filter shapes (`{ routeId, date, directionId, … }`) and returned GTFS / GTFS-RT object shapes are identical. No SQL query changes, no schema changes. +- The high-level entry points (`fromZip`, `fromZipData`, `fromDatabase`) keep their names and argument order; only `options` gains `adapter`. + +#### Step 1 — install the adapter peer dependency + +The core package no longer depends on sql.js. Install whichever adapter(s) you use: + +```bash +# Previously (v0.5 and earlier): already transitive — nothing to do. +# Now: +npm install sql.js # browser / Node WASM +npm install better-sqlite3 # Node native, file-backed +``` + +#### Step 2 — pass an adapter and `await` your queries + +Typical sql.js migration: + +```diff +- import { GtfsSqlJs } from 'gtfs-sqljs'; ++ import { GtfsSqlJs } from 'gtfs-sqljs'; ++ import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; + +- const gtfs = await GtfsSqlJs.fromZip(url, { locateFile }); ++ const gtfs = await GtfsSqlJs.fromZip(url, { ++ adapter: await createSqlJsAdapter({ locateFile }), ++ }); + +- const routes = gtfs.getRoutes(); +- const stops = gtfs.getStops({ name: 'Station' }); ++ const routes = await gtfs.getRoutes(); ++ const stops = await gtfs.getStops({ name: 'Station' }); + +- const buffer = gtfs.export(); +- gtfs.close(); ++ const buffer = await gtfs.export(); ++ await gtfs.close(); +``` + +TypeScript flags the missing `await`s for you; plain JS does not — grep for `gtfs.get` and `gtfs.close(`/`gtfs.export(` before shipping. + +#### Step 3 — only if you called `getDatabase()` for raw access + +`getDatabase()` now returns a `GtfsDatabase` (the adapter surface), not a raw sql.js `Database`. **All its methods are async.** This is the most likely silent failure during migration: + +```diff + const db = gtfs.getDatabase(); +- const stmt = db.prepare('SELECT * FROM stops WHERE stop_lat > ?'); +- stmt.bind([40.7]); +- while (stmt.step()) { +- const row = stmt.getAsObject(); ++ const stmt = await db.prepare('SELECT * FROM stops WHERE stop_lat > ?'); ++ await stmt.bind([40.7]); ++ while (await stmt.step()) { ++ const row = await stmt.getAsObject(); + console.log(row); + } +- stmt.free(); ++ await stmt.free(); +``` + +If you need the genuine sql.js `Database` (for features gtfs-sqljs does not wrap), keep a reference to it at the point where you built the adapter — the library no longer re-exposes it. + +#### Option A — new `attach()` entry point + +If you already open a database handle yourself (typical for file-backed drivers), skip the factory and attach the handle directly: + +```ts +import BetterSqlite3 from 'better-sqlite3'; +import { GtfsSqlJs } from 'gtfs-sqljs'; +import { wrapBetterSqlite3 } from 'gtfs-sqljs/adapters/better-sqlite3'; + +const raw = new BetterSqlite3('./gtfs.db', { readonly: true }); +const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw), { + skipSchema: true, // file already has the GTFS schema +}); +``` + +`attach()` does not take an `adapter`. By default it does **not** close the raw handle when `gtfs.close()` runs — pass `ownsDatabase: true` if you want the library to own it. + +#### Option B — removed / renamed options + +| v0.5 | v0.6 | +| --- | --- | +| `GtfsSqlJsOptions.SQL` | `createSqlJsAdapter({ SQL })` | +| `GtfsSqlJsOptions.locateFile` | `createSqlJsAdapter({ locateFile })` | +| re-exported `SqlJsStatic`, sql.js `Database` type | import from `sql.js` directly, or use `GtfsDatabase` | + +Calling `fromZip` / `fromZipData` / `fromDatabase` without `options.adapter` now throws a runtime `Error` pointing at `createSqlJsAdapter` — useful when you miss a call site. + +### New features + +- New `src/adapters/types.ts` public surface: `GtfsDatabase`, `GtfsStatement`, `GtfsDatabaseAdapter`, `SqlValue`, `Row`, `ExportNotSupportedError`. +- sql.js adapter at subpath `gtfs-sqljs/adapters/sql-js` (exports `createSqlJsAdapter`, `wrapSqlJsDatabase`). The core module no longer imports sql.js. +- **better-sqlite3 adapter at subpath `gtfs-sqljs/adapters/better-sqlite3`** (exports `wrapBetterSqlite3`, `createBetterSqlite3Adapter`). First-class Node / file-backed path; the adapter is the only file in the repo that imports `better-sqlite3`, so projects that do not reference this subpath never pull in the native module. Exercised by `tests/e2e-better-sqlite3.test.ts` on every CI run. +- Cache layer now catches `ExportNotSupportedError` from adapters that cannot serialize in-memory and logs a warning instead of failing the load; file-backed drivers persist their own DB on disk. + +### Performance (from earlier work in this cycle) - Ingestion is ~35-45% faster on medium-to-large feeds: ASTUCE (Rouen, ~430k stop_times rows) drops from ~2650 ms to ~1670 ms; Car Jaune from ~312 ms to ~188 ms. Wins come from parsing each CSV only once (progress totals now use a fast newline-based row-count estimate), loading rows as positional arrays instead of per-row objects, and reusing a single prepared INSERT per table instead of re-preparing a multi-row statement per 1000-row batch. -- Dropped the bulk-load PRAGMA block (`synchronous`, `journal_mode`, `temp_store`, `cache_size`, `locking_mode`) from ingestion. Benchmarked aggregate effect on sql.js is within noise (≤1%); removing them simplifies the code and unblocks upcoming pluggable-adapter work. +- Dropped the bulk-load PRAGMA block (`synchronous`, `journal_mode`, `temp_store`, `cache_size`, `locking_mode`) from ingestion. Benchmarked aggregate effect on sql.js is within noise (≤1%); removing them simplifies the code and unblocks the pluggable adapter. ### Behaviour changes diff --git a/README.md b/README.md index d875fb0..b4be3e7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![npm version](https://img.shields.io/npm/v/gtfs-sqljs)](https://www.npmjs.com/package/gtfs-sqljs) -

A TypeScript library for loading GTFS (General Transit Feed Specification) data into a sql.js SQLite database for querying in both browser and Node.js environments.

+

A TypeScript library for loading GTFS (General Transit Feed Specification) data into a SQLite database for querying in browser, Node.js, and React Native environments. Ships with adapters for sql.js (browser / Node WASM) and better-sqlite3 (Node native); bring your own for op-sqlite, expo-sqlite, etc.

> **[Live Demo](https://sysdevrun.github.io/gtfs-sqljs-demo/)** — A fully static demo website with GTFS and GTFS-RT data running in a Web Worker, with no backend. @@ -28,15 +28,16 @@ This project is greatly inspired by [node-gtfs](https://github.com/BlinkTagInc/n ## Features ### GTFS Static Data -- Load GTFS data from ZIP files (URL or local path) +- Load GTFS data from ZIP files (URL or `ArrayBuffer`) or existing SQLite databases +- **Pluggable database adapter** — sql.js, better-sqlite3 (built-in), or your own for op-sqlite / expo-sqlite +- **Attach to a pre-opened database** — ideal for file-backed native drivers where the caller controls the file path, readonly flag, etc. - **High-performance loading** with optimized bulk inserts - **Progress tracking** - Real-time progress callbacks (0-100%) - Skip importing specific files (e.g., shapes.txt) to reduce memory usage -- Load existing SQLite databases -- Export databases to ArrayBuffer for persistence +- Export databases to `ArrayBuffer` for persistence (sql.js / in-memory better-sqlite3) - Flexible filter-based query API - combine multiple filters easily - Full TypeScript support with comprehensive types -- Works in both browser and Node.js +- Works in browser, Node.js, and React Native ### [GTFS Realtime](https://gtfs.org/documentation/realtime/reference/) Support - Load GTFS-RT data from protobuf feeds (URLs or local files) @@ -56,51 +57,99 @@ This project is greatly inspired by [node-gtfs](https://github.com/BlinkTagInc/n npm install gtfs-sqljs ``` -You also need to install sql.js as a peer dependency: +Install the adapter(s) you want as peer dependencies. Install one or both depending on where the library runs: ```bash +# Browser or Node (WASM-backed, in-memory) npm install sql.js + +# Node (native, can be file-backed) +npm install better-sqlite3 ``` +> **Note (v0.6 breaking change):** the core library no longer hard-depends on sql.js. You must pass an adapter to `fromZip` / `fromZipData` / `fromDatabase`, or hand a pre-opened handle to `GtfsSqlJs.attach()`. All query methods are now `async` and return `Promise`. + ## Quick Start +### sql.js (browser / Node WASM) + ```typescript import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; -// Load GTFS data from a ZIP file -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip'); +// Load GTFS data from a ZIP URL +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter(), +}); // Query routes -const routes = gtfs.getRoutes(); +const routes = await gtfs.getRoutes(); // Query stops with filters -const stops = gtfs.getStops({ name: 'Central Station' }); +const stops = await gtfs.getStops({ name: 'Central Station' }); // Get trips for a route on a specific date -const trips = gtfs.getTrips({ +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115', - directionId: 0 + directionId: 0, }); // Get stop times for a trip -const stopTimes = gtfs.getStopTimes({ tripId: trips[0].trip_id }); +const stopTimes = await gtfs.getStopTimes({ tripId: trips[0].trip_id }); // Clean up -gtfs.close(); +await gtfs.close(); +``` + +### better-sqlite3 (Node native) + +```typescript +import BetterSqlite3 from 'better-sqlite3'; +import { GtfsSqlJs } from 'gtfs-sqljs'; +import { wrapBetterSqlite3 } from 'gtfs-sqljs/adapters/better-sqlite3'; + +// Open a file-backed DB yourself, then attach. +const raw = new BetterSqlite3('./gtfs.db'); +const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw)); + +const routes = await gtfs.getRoutes(); + +// `attach()` does not own the handle by default — you close both. +await gtfs.close(); +raw.close(); ``` -For detailed usage examples, see the [Usage Guide](https://sysdevrun.github.io/gtfs-sqljs/docs/documents/Usage_Guide.html). +See the [Usage Guide](https://sysdevrun.github.io/gtfs-sqljs/docs/documents/Usage_Guide.html) for detailed examples covering `fromDatabase`, `fromZipData`, GTFS-RT, and caching. + +## Adapters + +gtfs-sqljs talks to a narrow async `GtfsDatabase` interface (`prepare`, `run`, `export`, `close`). Pick the adapter that matches your runtime: + +| Adapter | Subpath | Typical use | +|---|---|---| +| sql.js | `gtfs-sqljs/adapters/sql-js` | Browser; Node without native deps; always in-memory | +| better-sqlite3 | `gtfs-sqljs/adapters/better-sqlite3` | Node; file-backed persistence; fastest native performance | +| op-sqlite | *(user-provided — see Usage Guide)* | React Native (JSI) | +| expo-sqlite | *(user-provided — see Usage Guide)* | Expo / React Native | + +Two entry points cover every scenario: + +- **Factory path** — `fromZip`/`fromZipData`/`fromDatabase` take `options.adapter: GtfsDatabaseAdapter`. The library creates / opens the DB for you. Best for in-memory drivers (sql.js, in-memory better-sqlite3). +- **Pre-opened handle** — `GtfsSqlJs.attach(db, options?)` takes a live `GtfsDatabase` you already built. Best for file-backed drivers where the caller owns the file path, journal mode, etc. ## API Reference Full API documentation: [API Reference](https://sysdevrun.github.io/gtfs-sqljs/docs/) +All `GtfsSqlJs` instance methods return `Promise` — use `await`. + ### Static Methods -- `GtfsSqlJs.fromZip(zipPath, options?)` - Create instance from GTFS ZIP file path or URL -- `GtfsSqlJs.fromZipData(zipData, options?)` - Create instance from pre-loaded GTFS ZIP data (`ArrayBuffer` or `Uint8Array`) -- `GtfsSqlJs.fromDatabase(database, options?)` - Create instance from existing database +- `GtfsSqlJs.fromZip(zipPath, options)` — Create instance from a GTFS ZIP URL. `options.adapter` is **required**. +- `GtfsSqlJs.fromZipData(zipData, options)` — Create instance from pre-loaded ZIP bytes (`ArrayBuffer` or `Uint8Array`). `options.adapter` is **required**. +- `GtfsSqlJs.fromDatabase(database, options)` — Create instance from existing SQLite bytes (`ArrayBuffer`). `options.adapter` is **required**. +- `GtfsSqlJs.attach(db, options?)` — Attach to a pre-opened `GtfsDatabase` handle. No `adapter` needed (the handle is the adapter output). Pass `skipSchema: true` when the attached DB already has the GTFS schema; pass `ownsDatabase: true` to have `close()` release the underlying handle. ### Instance Methods @@ -152,6 +201,8 @@ This library is written in TypeScript and provides full type definitions for all ```typescript import type { + // Adapter surface + GtfsDatabase, GtfsDatabaseAdapter, GtfsStatement, Row, SqlValue, // Static GTFS types Stop, Route, Trip, StopTime, Shape, TripFilters, StopTimeFilters, ShapeFilters, @@ -165,6 +216,7 @@ import type { // Progress tracking types ProgressInfo, ProgressCallback } from 'gtfs-sqljs'; +import { ExportNotSupportedError } from 'gtfs-sqljs'; ``` ## GTFS Specification diff --git a/documents/guide.md b/documents/guide.md index b45ae19..599105c 100644 --- a/documents/guide.md +++ b/documents/guide.md @@ -6,101 +6,242 @@ title: Usage Guide This guide covers all usage patterns for **gtfs-sqljs**, from basic setup to advanced features like GTFS Realtime and smart caching. +> **Breaking change in v0.6.** The library no longer hard-depends on sql.js. Pass an adapter explicitly via `options.adapter`, or hand a pre-opened handle to `GtfsSqlJs.attach()`. All query methods now return `Promise` — use `await`. + +## Adapters overview + +gtfs-sqljs talks to a small async database interface (`GtfsDatabase`). Two adapters ship in the box: + +| Subpath | Driver | When to use | +|---|---|---| +| `gtfs-sqljs/adapters/sql-js` | [sql.js](https://sql.js.org/) (WASM) | Browser; Node without native deps; always in-memory. | +| `gtfs-sqljs/adapters/better-sqlite3` | [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) (native) | Node; file-backed persistence; fastest native performance. | + +Two entry points cover every scenario: + +- **Factory path** — `fromZip` / `fromZipData` / `fromDatabase` take `options.adapter`. The library creates or opens the DB internally. Best for in-memory drivers. +- **Pre-opened handle** — `GtfsSqlJs.attach(db, options?)` takes a live `GtfsDatabase` you built yourself. Best for file-backed drivers where you want to control the file path, journal mode, readonly flag, etc. + ## Loading the sql.js WASM File -sql.js requires a WASM file to be loaded. There are several ways to handle this: +sql.js requires a WASM file. You configure it via `createSqlJsAdapter({ locateFile })`. ### Node.js -In Node.js, sql.js will automatically locate the WASM file from the installed package: +sql.js locates its WASM automatically in Node.js. No extra setup needed: ```typescript import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; -// The WASM file is loaded automatically -const gtfs = await GtfsSqlJs.fromZip('path/to/gtfs.zip'); +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter(), +}); ``` ### Browser with CDN -You can use a CDN to serve the WASM file: - ```typescript -import initSqlJs from 'sql.js'; import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; -// Initialize sql.js with CDN WASM file -const SQL = await initSqlJs({ - locateFile: (filename) => `https://sql.js.org/dist/${filename}` +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter({ + locateFile: (filename) => `https://sql.js.org/dist/${filename}`, + }), }); - -// Pass the SQL instance to GtfsSqlJs -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { SQL }); ``` -### Browser with Bundler (Webpack, Vite, etc.) - -If you're using a bundler, you need to configure it to handle the WASM file: +### Browser with Bundler (Vite / Webpack / …) #### Vite ```typescript -import initSqlJs from 'sql.js'; import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; import sqlWasmUrl from 'sql.js/dist/sql-wasm.wasm?url'; -const SQL = await initSqlJs({ - locateFile: () => sqlWasmUrl +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter({ locateFile: () => sqlWasmUrl }), }); - -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { SQL }); ``` #### Webpack +```typescript +import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; + +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter({ + locateFile: (filename) => `/path/to/public/${filename}`, + }), +}); +``` + +Copy `sql-wasm.wasm` from `node_modules/sql.js/dist/` to your public directory. + +### Reusing an existing `SqlJsStatic` + +If you already called `initSqlJs()` elsewhere, pass the instance in to skip re-initialization: + ```typescript import initSqlJs from 'sql.js'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; + +const SQL = await initSqlJs({ locateFile: (f) => `/sqljs/${f}` }); +const adapter = await createSqlJsAdapter({ SQL }); +``` + +## Creating an Instance + +This section covers the four typical starting points: ZIP on disk or as bytes, an existing `.db` as bytes, and a pre-opened native handle. + +### sql.js — from a GTFS ZIP URL + +```typescript import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; -const SQL = await initSqlJs({ - locateFile: (filename) => `/path/to/public/${filename}` +const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter(), }); -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { SQL }); +// Skip importing specific files to reduce memory usage. +// Tables are still created; just no data is inserted for them. +const lean = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter(), + skipFiles: ['shapes.txt', 'frequencies.txt'], +}); ``` -Make sure to copy `sql-wasm.wasm` from `node_modules/sql.js/dist/` to your public directory. +### sql.js — from GTFS ZIP bytes (no fetch or no file path) -## Creating an Instance +If you already have the ZIP in memory (uploaded from a ``, pre-fetched, bundled as an asset…): + +```typescript +import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; + +const zipBytes: ArrayBuffer = /* from fetch / FileReader / fs.readFile / … */; + +const gtfs = await GtfsSqlJs.fromZipData(zipBytes, { + adapter: await createSqlJsAdapter(), +}); + +const routes = await gtfs.getRoutes(); +``` -### From a GTFS ZIP file +### sql.js — from an existing SQLite database + +Use this when you have a pre-built `.db` (e.g. an earlier `gtfs.export()` saved to disk or shipped as a static asset): ```typescript import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; -// From URL -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip'); +// Browser +const dbBytes = await fetch('https://example.com/gtfs.db').then(r => r.arrayBuffer()); -// From local file (Node.js) -const gtfs = await GtfsSqlJs.fromZip('./path/to/gtfs.zip'); +// Node +// const dbBytes = (await fs.readFile('./gtfs.db')).buffer; -// Skip importing specific files to reduce memory usage -// Tables will be created but data won't be imported -const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { - skipFiles: ['shapes.txt', 'frequencies.txt'] +const gtfs = await GtfsSqlJs.fromDatabase(dbBytes, { + adapter: await createSqlJsAdapter(), }); ``` -### From an existing SQLite database +### better-sqlite3 — attach to a pre-opened file + +The idiomatic better-sqlite3 pattern: **you** create the connection (path, readonly, pragmas…) and hand the wrapped handle to `attach()`. ```typescript +import BetterSqlite3 from 'better-sqlite3'; import { GtfsSqlJs } from 'gtfs-sqljs'; +import { wrapBetterSqlite3 } from 'gtfs-sqljs/adapters/better-sqlite3'; -// Load from ArrayBuffer -const dbBuffer = await fetch('https://example.com/gtfs.db').then(r => r.arrayBuffer()); -const gtfs = await GtfsSqlJs.fromDatabase(dbBuffer); +// Read-only attach to an existing GTFS DB on disk. +const raw = new BetterSqlite3('./gtfs.db', { readonly: true }); +const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw), { + skipSchema: true, // the file already has the GTFS schema +}); + +const routes = await gtfs.getRoutes(); + +// attach() does not own the handle by default — you close both. +await gtfs.close(); +raw.close(); +``` + +If the file is **empty** or you want the library to create the schema, omit `skipSchema`: + +```typescript +const raw = new BetterSqlite3('./gtfs.db'); +const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw)); +// CREATE TABLE IF NOT EXISTS … runs automatically. +``` + +Pass `ownsDatabase: true` if you want `gtfs.close()` to also close the raw handle: + +```typescript +const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw), { ownsDatabase: true }); +// Later: await gtfs.close(); // raw is closed too +``` + +### better-sqlite3 — factory path (in-memory) + +Use the factory when you want the library to manage an in-memory better-sqlite3 DB — typical for ingesting a GTFS ZIP you have in memory without writing anything to disk: + +```typescript +import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createBetterSqlite3Adapter } from 'gtfs-sqljs/adapters/better-sqlite3'; + +const zipBytes: ArrayBuffer = /* … */; + +const gtfs = await GtfsSqlJs.fromZipData(zipBytes, { + adapter: createBetterSqlite3Adapter(), // defaults to ':memory:' +}); + +const trips = await gtfs.getTrips({ routeId: 'AB' }); ``` +Pass a path to create / open a file-backed DB via the factory: + +```typescript +const gtfs = await GtfsSqlJs.fromZipData(zipBytes, { + adapter: createBetterSqlite3Adapter('./gtfs.db'), +}); +``` + +### Decision table + +| You have… | sql.js | better-sqlite3 | +|---|---|---| +| A GTFS **ZIP URL** | `fromZip(url, { adapter: await createSqlJsAdapter(...) })` | `fromZip(url, { adapter: createBetterSqlite3Adapter(path?) })` | +| GTFS **ZIP bytes** | `fromZipData(zip, { adapter: await createSqlJsAdapter(...) })` | `fromZipData(zip, { adapter: createBetterSqlite3Adapter(path?) })` | +| An **existing `.db` as bytes** | `fromDatabase(bytes, { adapter: await createSqlJsAdapter(...) })` | Write the bytes to a file first, then `attach()` — better-sqlite3 opens paths, not buffers, for file-backed use | +| A **pre-opened handle** you control | *(uncommon for sql.js)* | `attach(wrapBetterSqlite3(raw), { skipSchema? })` | + +### Bringing your own adapter (op-sqlite / expo-sqlite / …) + +Implement the `GtfsDatabase` / `GtfsDatabaseAdapter` interfaces from `gtfs-sqljs`. The full contract is small — `prepare`, `run`, `export`, `close` on the DB, and `bind`, `step`, `getAsObject`, `run`, `free` on statements. File-backed drivers that cannot serialize should throw `ExportNotSupportedError` from `export()`; the cache layer catches it and no-ops. + +```typescript +import type { GtfsDatabase, GtfsDatabaseAdapter, GtfsStatement, Row, SqlValue } from 'gtfs-sqljs'; +import { ExportNotSupportedError } from 'gtfs-sqljs'; + +function wrapMyDriver(raw: MyDriverDb): GtfsDatabase { + return { + prepare: async (sql) => /* … wrap into GtfsStatement … */, + run: async (sql) => { raw.exec(sql); }, + export: async () => { throw new ExportNotSupportedError(); }, + close: async () => { raw.close(); }, + }; +} +``` + +See `src/adapters/better-sqlite3/index.ts` in the repository for a complete reference implementation (~130 lines). + ## Progress Tracking Track loading progress with a callback function - perfect for displaying progress bars or updating UI: @@ -206,17 +347,17 @@ The library provides flexible filter-based methods for querying GTFS data. Pass ```typescript // Get stops - combine any filters -const stops = gtfs.getStops({ +const stops = await gtfs.getStops({ name: 'Station', // Search by name limit: 10 // Limit results }); // Get routes - with or without filters -const allRoutes = gtfs.getRoutes(); -const agencyRoutes = gtfs.getRoutes({ agencyId: 'AGENCY_1' }); +const allRoutes = await gtfs.getRoutes(); +const agencyRoutes = await gtfs.getRoutes({ agencyId: 'AGENCY_1' }); // Get trips - combine multiple filters -const trips = gtfs.getTrips({ +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1', // Filter by route date: '20240115', // Filter by date (gets active services) directionId: 0, // Filter by direction @@ -224,7 +365,7 @@ const trips = gtfs.getTrips({ }); // Get stop times - flexible filtering -const stopTimes = gtfs.getStopTimes({ +const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP_123', // At a specific stop routeId: 'ROUTE_1', // For a specific route date: '20240115', // On a specific date @@ -275,122 +416,125 @@ const stopTimes = gtfs.getStopTimes({ ```typescript // Get stop by ID -const stops = gtfs.getStops({ stopId: 'STOP_123' }); +const stops = await gtfs.getStops({ stopId: 'STOP_123' }); const stop = stops.length > 0 ? stops[0] : null; console.log(stop?.stop_name); // Get stop by code (using filters) -const stops = gtfs.getStops({ stopCode: 'ABC' }); +const stops = await gtfs.getStops({ stopCode: 'ABC' }); const stop = stops[0]; // Search stops by name (using filters) -const stops = gtfs.getStops({ name: 'Main Street' }); +const stops = await gtfs.getStops({ name: 'Main Street' }); // Get all stops (using filters with no parameters) -const allStops = gtfs.getStops(); +const allStops = await gtfs.getStops(); // Get stops with limit -const stops = gtfs.getStops({ limit: 10 }); +const stops = await gtfs.getStops({ limit: 10 }); // Get stops for a specific trip -const stops = gtfs.getStops({ tripId: 'TRIP_123' }); +const stops = await gtfs.getStops({ tripId: 'TRIP_123' }); ``` ### Get Route Information ```typescript // Get route by ID -const routes = gtfs.getRoutes({ routeId: 'ROUTE_1' }); +const routes = await gtfs.getRoutes({ routeId: 'ROUTE_1' }); const route = routes.length > 0 ? routes[0] : null; // Get all routes (using filters with no parameters) -const routes = gtfs.getRoutes(); +const routes = await gtfs.getRoutes(); // Get routes by agency (using filters) -const agencyRoutes = gtfs.getRoutes({ agencyId: 'AGENCY_1' }); +const agencyRoutes = await gtfs.getRoutes({ agencyId: 'AGENCY_1' }); // Get routes with limit -const routes = gtfs.getRoutes({ limit: 10 }); +const routes = await gtfs.getRoutes({ limit: 10 }); ``` ### Get Agency Information ```typescript // Get agency by ID -const agencies = gtfs.getAgencies({ agencyId: 'AGENCY_1' }); +const agencies = await gtfs.getAgencies({ agencyId: 'AGENCY_1' }); const agency = agencies.length > 0 ? agencies[0] : null; // Get all agencies -const allAgencies = gtfs.getAgencies(); +const allAgencies = await gtfs.getAgencies(); // Get agencies with limit -const agencies = gtfs.getAgencies({ limit: 5 }); +const agencies = await gtfs.getAgencies({ limit: 5 }); ``` ### Get Calendar Information ```typescript // Get active services for a date (YYYYMMDD format) -const serviceIds = gtfs.getActiveServiceIds('20240115'); +const serviceIds = await gtfs.getActiveServiceIds('20240115'); + +// Get calendar by service ID (returns Calendar | null) +const calendar = await gtfs.getCalendarByServiceId('WEEKDAY'); -// Get calendar by service ID -const calendars = gtfs.getCalendars({ serviceId: 'WEEKDAY' }); +// Get calendar date exceptions for a service +const exceptions = await gtfs.getCalendarDates('WEEKDAY'); -// Get calendar date exceptions -const exceptions = gtfs.getCalendarDates('WEEKDAY'); +// Get calendar date exceptions for a specific date +const exceptionsForDate = await gtfs.getCalendarDatesForDate('20240115'); ``` ### Get Trip Information ```typescript // Get trip by ID -const trips = gtfs.getTrips({ tripId: 'TRIP_123' }); +const trips = await gtfs.getTrips({ tripId: 'TRIP_123' }); const trip = trips.length > 0 ? trips[0] : null; // Get trips by route (using filters) -const trips = gtfs.getTrips({ routeId: 'ROUTE_1' }); +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1' }); // Get trips by route and date (using filters) -const trips = gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115' }); +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115' }); // Get trips by route, date, and direction (using filters) -const trips = gtfs.getTrips({ +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115', directionId: 0 }); // Get all trips for a date -const trips = gtfs.getTrips({ date: '20240115' }); +const trips = await gtfs.getTrips({ date: '20240115' }); // Get trips by agency -const trips = gtfs.getTrips({ agencyId: 'AGENCY_1' }); +const trips = await gtfs.getTrips({ agencyId: 'AGENCY_1' }); ``` ### Get Stop Time Information ```typescript // Get stop times for a trip (ordered by stop_sequence) -const stopTimes = gtfs.getStopTimes({ tripId: 'TRIP_123' }); +const stopTimes = await gtfs.getStopTimes({ tripId: 'TRIP_123' }); // Get stop times for a stop (using filters) -const stopTimes = gtfs.getStopTimes({ stopId: 'STOP_123' }); +const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP_123' }); // Get stop times for a stop and route (using filters) -const stopTimes = gtfs.getStopTimes({ +const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP_123', routeId: 'ROUTE_1' }); // Get stop times for a stop, route, and date (using filters) -const stopTimes = gtfs.getStopTimes({ +const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP_123', routeId: 'ROUTE_1', date: '20240115' }); // Get stop times with direction filter (using filters) -const stopTimes = gtfs.getStopTimes({ +const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP_123', routeId: 'ROUTE_1', date: '20240115', @@ -398,7 +542,7 @@ const stopTimes = gtfs.getStopTimes({ }); // Get stop times by agency -const stopTimes = gtfs.getStopTimes({ +const stopTimes = await gtfs.getStopTimes({ agencyId: 'AGENCY_1', date: '20240115' }); @@ -410,7 +554,7 @@ When displaying timetables for routes where different trips may stop at differen ```typescript // Get all trips for a route in one direction -const trips = gtfs.getTrips({ +const trips = await gtfs.getTrips({ routeId: 'ROUTE_1', directionId: 0, date: '20240115' @@ -418,7 +562,7 @@ const trips = gtfs.getTrips({ // Build ordered list of all stops served by these trips const tripIds = trips.map(t => t.trip_id); -const orderedStops = gtfs.buildOrderedStopList(tripIds); +const orderedStops = await gtfs.buildOrderedStopList(tripIds); // Now display a timetable with all possible stops console.log('Route stops:'); @@ -428,7 +572,7 @@ orderedStops.forEach(stop => { // For each trip, you can now show which stops it serves for (const trip of trips) { - const tripStopTimes = gtfs.getStopTimes({ tripId: trip.trip_id }); + const tripStopTimes = await gtfs.getStopTimes({ tripId: trip.trip_id }); console.log(`\nTrip ${trip.trip_headsign}:`); // Show all stops, marking which ones this trip serves @@ -463,9 +607,9 @@ The method intelligently merges stop sequences from all provided trips: // - Express trips: A -> C -> E -> F (skips B and D) // - Short trips: B -> C -> D (doesn't go to end of line) -const allTrips = gtfs.getTrips({ routeId: 'BUS_42', directionId: 0 }); +const allTrips = await gtfs.getTrips({ routeId: 'BUS_42', directionId: 0 }); const tripIds = allTrips.map(t => t.trip_id); -const stops = gtfs.buildOrderedStopList(tripIds); +const stops = await gtfs.buildOrderedStopList(tripIds); // Result: [A, B, C, D, E, F] - all stops in correct order // Now you can create a timetable showing all stops with departure times @@ -477,14 +621,14 @@ Shapes define the path a vehicle takes along a route. Use {@link GtfsSqlJs.getSh ```typescript // Get all shape points for a specific shape -const shapePoints = gtfs.getShapes({ shapeId: 'SHAPE_1' }); +const shapePoints = await gtfs.getShapes({ shapeId: 'SHAPE_1' }); console.log(`Shape has ${shapePoints.length} points`); // Get shapes for a specific route -const routeShapes = gtfs.getShapes({ routeId: 'ROUTE_1' }); +const routeShapes = await gtfs.getShapes({ routeId: 'ROUTE_1' }); // Get shapes for multiple trips -const tripShapes = gtfs.getShapes({ tripId: ['TRIP_1', 'TRIP_2'] }); +const tripShapes = await gtfs.getShapes({ tripId: ['TRIP_1', 'TRIP_2'] }); // Each shape point contains: // - shape_id: string @@ -500,13 +644,13 @@ Convert shapes to GeoJSON format for use with mapping libraries (Leaflet, Mapbox ```typescript // Get all shapes as GeoJSON FeatureCollection -const geojson = gtfs.getShapesToGeojson(); +const geojson = await gtfs.getShapesToGeojson(); // Get shapes for a specific route -const routeGeojson = gtfs.getShapesToGeojson({ routeId: 'ROUTE_1' }); +const routeGeojson = await gtfs.getShapesToGeojson({ routeId: 'ROUTE_1' }); // Customize coordinate precision (default: 6 decimals = ~10cm) -const lowPrecision = gtfs.getShapesToGeojson({ routeId: 'ROUTE_1' }, 4); // ~11m precision +const lowPrecision = await gtfs.getShapesToGeojson({ routeId: 'ROUTE_1' }, 4); // ~11m precision // GeoJSON structure: // { @@ -554,6 +698,7 @@ This library supports [GTFS Realtime](https://gtfs.org/documentation/realtime/re ```typescript // Configure RT feed URLs - data will be fetched automatically after GTFS load const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { + adapter: await createSqlJsAdapter(), realtimeFeedUrls: [ 'https://example.com/gtfs-rt/alerts', 'https://example.com/gtfs-rt/trip-updates', @@ -592,27 +737,27 @@ if (lastFetch) { ```typescript // Get all active alerts -const activeAlerts = gtfs.getAlerts({ activeOnly: true }); +const activeAlerts = await gtfs.getAlerts({ activeOnly: true }); // Filter alerts by route -const routeAlerts = gtfs.getAlerts({ +const routeAlerts = await gtfs.getAlerts({ routeId: 'ROUTE_1', activeOnly: true }); // Filter alerts by stop -const stopAlerts = gtfs.getAlerts({ +const stopAlerts = await gtfs.getAlerts({ stopId: 'STOP_123', activeOnly: true }); // Filter alerts by trip -const tripAlerts = gtfs.getAlerts({ +const tripAlerts = await gtfs.getAlerts({ tripId: 'TRIP_456' }); // Get alert by ID -const alerts = gtfs.getAlerts({ alertId: 'alert:12345' }); +const alerts = await gtfs.getAlerts({ alertId: 'alert:12345' }); const alert = alerts.length > 0 ? alerts[0] : null; // Alert structure @@ -628,15 +773,15 @@ console.log(alert.informed_entity); // EntitySelector[] ```typescript // Get all vehicle positions -const vehicles = gtfs.getVehiclePositions(); +const vehicles = await gtfs.getVehiclePositions(); // Filter by route -const routeVehicles = gtfs.getVehiclePositions({ +const routeVehicles = await gtfs.getVehiclePositions({ routeId: 'ROUTE_1' }); // Filter by trip -const tripVehicles = gtfs.getVehiclePositions({ +const tripVehicles = await gtfs.getVehiclePositions({ tripId: 'TRIP_123' }); const vehicle = tripVehicles.length > 0 ? tripVehicles[0] : null; @@ -654,7 +799,7 @@ The library automatically merges realtime data with static schedules when reques ```typescript // Get trips with realtime data -const tripsWithRT = gtfs.getTrips({ +const tripsWithRT = await gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115', includeRealtime: true // Include RT data @@ -670,7 +815,7 @@ for (const trip of tripsWithRT) { } // Get stop times with realtime delays -const stopTimesWithRT = gtfs.getStopTimes({ +const stopTimesWithRT = await gtfs.getStopTimes({ tripId: 'TRIP_123', includeRealtime: true // Include RT data }); @@ -688,7 +833,7 @@ for (const st of stopTimesWithRT) { ```typescript // Clear all realtime data -gtfs.clearRealtimeData(); +await gtfs.clearRealtimeData(); // Then fetch fresh data await gtfs.fetchRealtimeData(); @@ -726,30 +871,38 @@ Cache store implementations are available in `examples/cache/`. Copy the appropr ```typescript // Copy examples/cache/IndexedDBCacheStore.ts to your project import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; import { IndexedDBCacheStore } from './IndexedDBCacheStore'; const cache = new IndexedDBCacheStore(); +const adapter = await createSqlJsAdapter(); // First load: processes GTFS zip file and caches the result -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { cache }); +const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { adapter, cache }); // Second load: uses cached database (much faster!) -const gtfs2 = await GtfsSqlJs.fromZip('gtfs.zip', { cache }); +const gtfs2 = await GtfsSqlJs.fromZip('gtfs.zip', { adapter, cache }); ``` **Node.js - FileSystem:** ```typescript // Copy examples/cache/FileSystemCacheStore.ts to your project import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; import { FileSystemCacheStore } from './FileSystemCacheStore'; const cache = new FileSystemCacheStore({ dir: './.cache/gtfs' }); -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { cache }); +const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { + adapter: await createSqlJsAdapter(), + cache, +}); ``` **Note:** `FileSystemCacheStore` uses Node.js built-in modules (`fs`, `path`, `os`) and is **NOT compatible** with browser or React Native environments. +**Caching with file-backed adapters (better-sqlite3, op-sqlite, …):** file-backed drivers persist their own database on disk, so the library's cache is redundant. If you plug in a file-backed adapter and enable the cache, the cache layer catches `ExportNotSupportedError` from `export()` and logs a warning instead of failing the load. + ### Cache Invalidation The cache is automatically invalidated when any of these change: @@ -764,14 +917,20 @@ The cache is automatically invalidated when any of these change: Use `cacheVersion` to control cache invalidation: ```typescript +const adapter = await createSqlJsAdapter(); + // Load with version 1.0 const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { - cacheVersion: '1.0' + adapter, + cache, + cacheVersion: '1.0', }); // Load with version 2.0 - will reprocess and create new cache const gtfs2 = await GtfsSqlJs.fromZip('gtfs.zip', { - cacheVersion: '2.0' + adapter, + cache, + cacheVersion: '2.0', }); ``` @@ -850,7 +1009,9 @@ await GtfsSqlJs.clearCache(cache); By default, caching is disabled. Simply omit the `cache` option: ```typescript -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip'); +const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { + adapter: await createSqlJsAdapter(), +}); // No caching - GTFS is processed fresh each time ``` @@ -860,7 +1021,9 @@ Change the default expiration time (default: 7 days): ```typescript const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { - cacheExpirationMs: 3 * 24 * 60 * 60 * 1000 // 3 days + adapter: await createSqlJsAdapter(), + cache, + cacheExpirationMs: 3 * 24 * 60 * 60 * 1000, // 3 days }); ``` @@ -898,14 +1061,17 @@ class RedisCacheStore implements CacheStore { } const cache = new RedisCacheStore(); -const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { cache }); +const gtfs = await GtfsSqlJs.fromZip('gtfs.zip', { + adapter: await createSqlJsAdapter(), + cache, +}); ``` ## Export Database ```typescript // Export to ArrayBuffer for storage (includes RT data) -const buffer = gtfs.export(); +const buffer = await gtfs.export(); // Save to file (Node.js) import fs from 'fs'; @@ -919,85 +1085,89 @@ fs.writeFileSync('gtfs.db', Buffer.from(buffer)); ### Direct Database Access -For advanced queries not covered by the API: +For advanced queries not covered by the API, use `getDatabase()` — it returns the adapter's `GtfsDatabase`, which exposes `prepare`, `run`, `export`, `close`. Every method is async: ```typescript const db = gtfs.getDatabase(); -const stmt = db.prepare('SELECT * FROM stops WHERE stop_lat > ? AND stop_lon < ?'); -stmt.bind([40.7, -74.0]); +const stmt = await db.prepare('SELECT * FROM stops WHERE stop_lat > ? AND stop_lon < ?'); +await stmt.bind([40.7, -74.0]); -while (stmt.step()) { - const row = stmt.getAsObject(); +while (await stmt.step()) { + const row = await stmt.getAsObject(); console.log(row); } -stmt.free(); +await stmt.free(); ``` +If you need the underlying raw driver (sql.js `Database`, better-sqlite3 `Database`, …) for driver-specific features, keep a reference to it at the point you created the adapter — gtfs-sqljs deliberately does not re-expose it, to keep the library driver-agnostic. + ### Close Database ```typescript // Close the database when done -gtfs.close(); +await gtfs.close(); ``` +For `GtfsSqlJs.attach()` with a handle you created yourself, `close()` does **not** release the underlying handle by default — you are responsible for closing it. Pass `ownsDatabase: true` to `attach()` if you want the library to close it for you. + ## Complete Example ```typescript import { GtfsSqlJs } from 'gtfs-sqljs'; +import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js'; async function example() { // Load GTFS data (skip shapes.txt to reduce memory usage) const gtfs = await GtfsSqlJs.fromZip('https://example.com/gtfs.zip', { - skipFiles: ['shapes.txt'] + adapter: await createSqlJsAdapter(), + skipFiles: ['shapes.txt'], }); // Find a stop using flexible filters - const stops = gtfs.getStops({ name: 'Central Station' }); + const stops = await gtfs.getStops({ name: 'Central Station' }); const stop = stops[0]; console.log(`Found stop: ${stop.stop_name}`); // Find routes serving this stop (via stop_times and trips) - const allStopTimes = gtfs.getStopTimes({ stopId: stop.stop_id }); - const routeIds = new Set( - allStopTimes.map(st => { - const trips = gtfs.getTrips({ tripId: st.trip_id }); - return trips.length > 0 ? trips[0].route_id : null; - }) - ); + const allStopTimes = await gtfs.getStopTimes({ stopId: stop.stop_id }); + const routeIds = new Set(); + for (const st of allStopTimes) { + const trips = await gtfs.getTrips({ tripId: st.trip_id }); + if (trips.length > 0) routeIds.add(trips[0].route_id); + } // Get route details for (const routeId of routeIds) { - if (!routeId) continue; - const routes = gtfs.getRoutes({ routeId }); + const routes = await gtfs.getRoutes({ routeId }); const route = routes.length > 0 ? routes[0] : null; console.log(`Route: ${route?.route_short_name} - ${route?.route_long_name}`); } // Get trips for a specific route on a date using flexible filters const today = '20240115'; // YYYYMMDD format - const trips = gtfs.getTrips({ + const trips = await gtfs.getTrips({ routeId: Array.from(routeIds)[0]!, - date: today + date: today, }); console.log(`Found ${trips.length} trips for today`); // Get stop times for a specific trip - const stopTimes = gtfs.getStopTimes({ tripId: trips[0].trip_id }); + const stopTimes = await gtfs.getStopTimes({ tripId: trips[0].trip_id }); console.log('Trip schedule:'); for (const st of stopTimes) { - const stops = gtfs.getStops({ stopId: st.stop_id }); - const stop = stops.length > 0 ? stops[0] : null; - console.log(` ${st.arrival_time} - ${stop?.stop_name}`); + const matched = await gtfs.getStops({ stopId: st.stop_id }); + const matchedStop = matched.length > 0 ? matched[0] : null; + console.log(` ${st.arrival_time} - ${matchedStop?.stop_name}`); } // Export database for later use - const buffer = gtfs.export(); + const buffer = await gtfs.export(); // ... save buffer to file or storage // Clean up - gtfs.close(); + await gtfs.close(); } example(); diff --git a/examples/basic-usage.ts b/examples/basic-usage.ts index 8e9dd0f..2a9c0d0 100644 --- a/examples/basic-usage.ts +++ b/examples/basic-usage.ts @@ -1,84 +1,78 @@ /** - * Basic Usage Example for gtfs-sqljs + * Basic Usage Example for gtfs-sqljs (v0.5+ async / pluggable-adapter API). * - * This example demonstrates how to load GTFS data and perform basic queries. + * This example shows the sql.js path. For a file-backed native driver + * (better-sqlite3, op-sqlite, expo-sqlite), open a connection yourself and + * pass the wrapped handle to `GtfsSqlJs.attach()` instead. */ import { GtfsSqlJs } from '../src/index'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; async function main() { console.log('Loading GTFS data...'); - // Load GTFS data from a ZIP file - // Replace with your GTFS feed URL or local path - const gtfs = await GtfsSqlJs.fromZip('path/to/gtfs.zip'); + const gtfs = await GtfsSqlJs.fromZip('path/to/gtfs.zip', { + adapter: await createSqlJsAdapter(), + }); console.log('GTFS data loaded successfully!\n'); - // Example 1: Find stops by name console.log('=== Example 1: Search for stops ==='); - const stops = gtfs.searchStopsByName('Station', 5); + const stops = await gtfs.getStops({ name: 'Station', limit: 5 }); console.log(`Found ${stops.length} stops:`); stops.forEach(stop => { console.log(` - ${stop.stop_name} (${stop.stop_id})`); }); console.log(); - // Example 2: Get all routes console.log('=== Example 2: List all routes ==='); - const routes = gtfs.getAllRoutes(5); + const routes = await gtfs.getRoutes({ limit: 5 }); console.log(`Found ${routes.length} routes:`); routes.forEach(route => { console.log(` - ${route.route_short_name}: ${route.route_long_name}`); }); console.log(); - // Example 3: Get active services for today console.log('=== Example 3: Active services for a date ==='); - const date = '20240115'; // YYYYMMDD format - const serviceIds = gtfs.getActiveServiceIds(date); + const date = '20240115'; + const serviceIds = await gtfs.getActiveServiceIds(date); console.log(`Active services on ${date}:`); serviceIds.forEach(serviceId => { console.log(` - ${serviceId}`); }); console.log(); - // Example 4: Get trips for a route on a specific date if (routes.length > 0) { console.log('=== Example 4: Trips for a route ==='); const route = routes[0]; - const trips = gtfs.getTripsByRouteAndDate(route.route_id, date); + const trips = await gtfs.getTrips({ routeId: route.route_id, date }); console.log(`Trips for route ${route.route_short_name} on ${date}:`); trips.slice(0, 5).forEach(trip => { console.log(` - ${trip.trip_id} (${trip.trip_headsign || 'No headsign'})`); }); console.log(); - // Example 5: Get stop times for a trip if (trips.length > 0) { console.log('=== Example 5: Stop times for a trip ==='); const trip = trips[0]; - const stopTimes = gtfs.getStopTimesByTrip(trip.trip_id); + const stopTimes = await gtfs.getStopTimes({ tripId: trip.trip_id }); + const stopsForTrip = await gtfs.getStops({ tripId: trip.trip_id }); + const stopById = new Map(stopsForTrip.map(s => [s.stop_id, s])); console.log(`Schedule for trip ${trip.trip_id}:`); stopTimes.forEach(st => { - const stop = gtfs.getStopById(st.stop_id); - console.log(` ${st.arrival_time} - ${stop?.stop_name}`); + console.log(` ${st.arrival_time} - ${stopById.get(st.stop_id)?.stop_name ?? st.stop_id}`); }); console.log(); } } - // Example 6: Export database for later use console.log('=== Example 6: Export database ==='); - const buffer = gtfs.export(); + const buffer = await gtfs.export(); console.log(`Database exported: ${buffer.byteLength} bytes`); - console.log('You can save this buffer and load it later with GtfsSqlJs.fromDatabase()'); - console.log(); - // Clean up - gtfs.close(); + await gtfs.close(); console.log('Done!'); } -// Run the example main().catch(console.error); diff --git a/package-lock.json b/package-lock.json index 41834f1..f17ca55 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,25 +1,27 @@ { "name": "gtfs-sqljs", - "version": "0.3.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gtfs-sqljs", - "version": "0.3.0", + "version": "0.4.1", "license": "MIT", "dependencies": { "jszip": "^3.10.1", "papaparse": "^5.5.3", - "protobufjs": "^8.0.0", - "sql.js": "1.13.0" + "protobufjs": "^8.0.0" }, "devDependencies": { "@eslint/js": "^9.39.3", + "@types/better-sqlite3": "^7.6.11", "@types/node": "^25.3.1", "@types/papaparse": "^5.5.0", "@types/sql.js": "^1.4.9", + "better-sqlite3": "^11.3.0", "eslint": "^9.39.3", + "sql.js": "1.13.0", "tsup": "^8.0.1", "typedoc": "^0.28.17", "typescript": "^5.9.3", @@ -28,6 +30,14 @@ }, "engines": { "node": ">=18.0.0" + }, + "peerDependencies": { + "sql.js": "1.x" + }, + "peerDependenciesMeta": { + "sql.js": { + "optional": true + } } }, "node_modules/@esbuild/aix-ppc64": { @@ -1250,6 +1260,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1772,6 +1792,76 @@ "node": "18 || 20 || >=22" } }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", + "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/brace-expansion": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.3.tgz", @@ -1785,6 +1875,31 @@ "node": "18 || 20 || >=22" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/bundle-require": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/bundle-require/-/bundle-require-5.1.0.tgz", @@ -1864,6 +1979,13 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1957,6 +2079,32 @@ } } }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1964,6 +2112,26 @@ "dev": true, "license": "MIT" }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -2272,6 +2440,16 @@ "node": ">=0.10.0" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/expect-type": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", @@ -2316,6 +2494,13 @@ "node": ">=16.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "license": "MIT" + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -2366,6 +2551,13 @@ "dev": true, "license": "ISC" }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -2381,6 +2573,13 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true, + "license": "MIT" + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2417,6 +2616,27 @@ "node": ">=8" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/ignore": { "version": "7.0.5", "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", @@ -2466,6 +2686,13 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "license": "ISC" }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, + "license": "ISC" + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -2702,6 +2929,19 @@ "dev": true, "license": "MIT" }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "10.2.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", @@ -2718,6 +2958,23 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, "node_modules/mlly": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", @@ -2769,6 +3026,13 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "dev": true, + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -2776,6 +3040,19 @@ "dev": true, "license": "MIT" }, + "node_modules/node-abi": { + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2797,6 +3074,16 @@ ], "license": "MIT" }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -3001,6 +3288,34 @@ } } }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -3041,6 +3356,17 @@ "node": ">=12.0.0" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -3061,6 +3387,32 @@ "node": ">=6" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -3200,6 +3552,53 @@ "dev": true, "license": "ISC" }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/source-map": { "version": "0.7.6", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.6.tgz", @@ -3224,6 +3623,7 @@ "version": "1.13.0", "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.13.0.tgz", "integrity": "sha512-RJbVP1HRDlUUXahJ7VMTcu9Rm1Nzw+EBpoPr94vnbD4LwR715F3CcxE2G2k45PewcaZ57pjetYa+LoSJLAASgA==", + "dev": true, "license": "MIT" }, "node_modules/stackback": { @@ -3298,6 +3698,51 @@ "node": ">=8" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tar-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -3487,6 +3932,19 @@ "node": ">=8" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -3867,6 +4325,13 @@ "node": ">=0.10.0" } }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, "node_modules/yaml": { "version": "2.8.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", diff --git a/package.json b/package.json index 20f1c86..7f352d6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "gtfs-sqljs", "version": "0.4.1", - "description": "Load GTFS data into sql.js SQLite database for querying in browser and Node.js", + "description": "Load GTFS data into a SQLite database (sql.js / better-sqlite3 / op-sqlite / expo-sqlite) via a pluggable adapter", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -9,6 +9,14 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./adapters/sql-js": { + "types": "./dist/adapters/sql-js/index.d.ts", + "default": "./dist/adapters/sql-js/index.js" + }, + "./adapters/better-sqlite3": { + "types": "./dist/adapters/better-sqlite3/index.d.ts", + "default": "./dist/adapters/better-sqlite3/index.js" } }, "files": [ @@ -32,8 +40,12 @@ "transit", "sqlite", "sql.js", + "better-sqlite3", + "op-sqlite", + "expo-sqlite", "browser", - "nodejs" + "nodejs", + "react-native" ], "author": "Théophile Helleboid/SysDevRun", "license": "MIT", @@ -44,15 +56,29 @@ "dependencies": { "jszip": "^3.10.1", "papaparse": "^5.5.3", - "protobufjs": "^8.0.0", - "sql.js": "1.13.0" + "protobufjs": "^8.0.0" + }, + "peerDependencies": { + "better-sqlite3": ">=10", + "sql.js": "1.x" + }, + "peerDependenciesMeta": { + "better-sqlite3": { + "optional": true + }, + "sql.js": { + "optional": true + } }, "devDependencies": { "@eslint/js": "^9.39.3", + "@types/better-sqlite3": "^7.6.11", "@types/node": "^25.3.1", "@types/papaparse": "^5.5.0", "@types/sql.js": "^1.4.9", + "better-sqlite3": "^11.3.0", "eslint": "^9.39.3", + "sql.js": "1.13.0", "tsup": "^8.0.1", "typedoc": "^0.28.17", "typescript": "^5.9.3", diff --git a/src/adapters/better-sqlite3/index.ts b/src/adapters/better-sqlite3/index.ts new file mode 100644 index 0000000..2dff33f --- /dev/null +++ b/src/adapters/better-sqlite3/index.ts @@ -0,0 +1,159 @@ +/** + * better-sqlite3 adapter for gtfs-sqljs. + * + * Ships as an opt-in subpath export (`gtfs-sqljs/adapters/better-sqlite3`). + * This is the only file in the repository that imports `better-sqlite3`, so + * projects that never reference this subpath do not pull in the native + * module. + * + * Typical usage (caller-managed DB, recommended for file-backed drivers): + * + * ```ts + * import Database from 'better-sqlite3'; + * import { GtfsSqlJs } from 'gtfs-sqljs'; + * import { wrapBetterSqlite3 } from 'gtfs-sqljs/adapters/better-sqlite3'; + * + * const raw = new Database('gtfs.db'); + * const gtfs = await GtfsSqlJs.attach(wrapBetterSqlite3(raw)); + * ``` + * + * Library-managed usage (in-memory, less common): + * + * ```ts + * import { GtfsSqlJs } from 'gtfs-sqljs'; + * import { createBetterSqlite3Adapter } from 'gtfs-sqljs/adapters/better-sqlite3'; + * + * const gtfs = await GtfsSqlJs.fromZip(url, { + * adapter: createBetterSqlite3Adapter(), + * }); + * ``` + */ + +import BetterSqlite3 from 'better-sqlite3'; +import { + ExportNotSupportedError, + type GtfsDatabase, + type GtfsDatabaseAdapter, + type GtfsStatement, + type Row, + type SqlValue, +} from '../types'; + +type BetterSqlite3Database = BetterSqlite3.Database; + +type StatementState = { + stmt: BetterSqlite3.Statement; + iterator: IterableIterator | null; + currentRow: Row | null; + boundParams: SqlValue[] | null; +}; + +function wrapStatement(db: BetterSqlite3Database, sql: string): GtfsStatement { + const state: StatementState = { + stmt: db.prepare(sql), + iterator: null, + currentRow: null, + boundParams: null, + }; + + const resetIterator = () => { + state.iterator = null; + state.currentRow = null; + }; + + return { + bind: async (params) => { + state.boundParams = params; + resetIterator(); + }, + step: async () => { + if (!state.iterator) { + const params = state.boundParams ?? []; + state.iterator = state.stmt.iterate(...toPositionalArgs(params)); + } + const next = state.iterator.next(); + if (next.done) { + state.currentRow = null; + return false; + } + state.currentRow = next.value as Row; + return true; + }, + getAsObject: async () => state.currentRow ?? {}, + run: async (params) => { + state.stmt.run(...toPositionalArgs(params ?? [])); + resetIterator(); + }, + free: async () => { + resetIterator(); + // better-sqlite3 statements are GC-managed; no explicit release needed. + }, + }; +} + +/** + * Wrap an already-open `better-sqlite3` database handle as a `GtfsDatabase`. + * Pair with `GtfsSqlJs.attach()` — the caller retains ownership of the raw + * handle unless `ownsDatabase: true` is passed. + */ +export function wrapBetterSqlite3(db: BetterSqlite3Database): GtfsDatabase { + return { + prepare: async (sql) => wrapStatement(db, sql), + run: async (sql) => { + db.exec(sql); + }, + export: async () => { + // `serialize()` only works on in-memory databases (better-sqlite3 ≥ 10). + // File-backed DBs persist themselves on disk; the library's cache layer + // catches this error and no-ops. + const serialize = (db as unknown as { serialize?: () => Buffer }).serialize; + if (typeof serialize === 'function') { + const buf = serialize.call(db); + return new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + } + throw new ExportNotSupportedError( + 'better-sqlite3 cannot serialize this database. Read the underlying ' + + 'file on disk directly to persist it.' + ); + }, + close: async () => { + db.close(); + }, + }; +} + +/** + * Factory for the library-managed path (`fromZip` / `fromZipData` / + * `fromDatabase`). `filename` defaults to `:memory:`; pass a path to have the + * adapter create / open a file-backed database. + */ +export function createBetterSqlite3Adapter( + filename: string = ':memory:', + options?: BetterSqlite3.Options +): GtfsDatabaseAdapter { + return { + createEmpty: async () => wrapBetterSqlite3(new BetterSqlite3(filename, options)), + openFromBuffer: async (buf) => { + // better-sqlite3 accepts a Buffer as the first argument and loads the + // database bytes into an in-memory connection. The typings still say + // `string`, so we cast through `unknown`. + const data = buf instanceof Uint8Array + ? Buffer.from(buf.buffer, buf.byteOffset, buf.byteLength) + : Buffer.from(new Uint8Array(buf)); + const db = new (BetterSqlite3 as unknown as new ( + b: Buffer, + o?: BetterSqlite3.Options + ) => BetterSqlite3Database)(data, options); + return wrapBetterSqlite3(db); + }, + }; +} + +function toPositionalArgs(params: SqlValue[]): unknown[] { + return params.map((p) => { + if (p instanceof Uint8Array) { + return Buffer.from(p.buffer, p.byteOffset, p.byteLength); + } + return p; + }); +} diff --git a/src/adapters/sql-js/index.ts b/src/adapters/sql-js/index.ts new file mode 100644 index 0000000..3fb67f9 --- /dev/null +++ b/src/adapters/sql-js/index.ts @@ -0,0 +1,73 @@ +/** + * sql.js adapter for gtfs-sqljs. + * + * This module is the only file in the repository that imports `sql.js`. It + * ships as an opt-in subpath export (`gtfs-sqljs/adapters/sql-js`) so that + * consumers picking a different driver (op-sqlite, expo-sqlite, + * better-sqlite3, …) never pay the cost of bundling sql.js. + */ + +import initSqlJs, { + type SqlJsStatic, + type Database as SqlJsDatabase, + type Statement as SqlJsStatement, +} from 'sql.js'; +import type { + GtfsDatabaseAdapter, + GtfsDatabase, + GtfsStatement, + Row, + SqlValue, +} from '../types'; + +export interface SqlJsAdapterOptions { + /** Optional pre-initialized SqlJsStatic. Skips `initSqlJs()` when provided. */ + SQL?: SqlJsStatic; + /** Optional WASM locator passed to `initSqlJs({ locateFile })`. */ + locateFile?: (filename: string) => string; +} + +function wrapStatement(stmt: SqlJsStatement): GtfsStatement { + return { + bind: async (params) => { + stmt.bind(params as SqlValue[]); + }, + step: async () => stmt.step(), + getAsObject: async () => stmt.getAsObject() as Row, + run: async (params) => { + stmt.run(params as SqlValue[] | undefined); + }, + free: async () => { + stmt.free(); + }, + }; +} + +export function wrapSqlJsDatabase(db: SqlJsDatabase): GtfsDatabase { + return { + prepare: async (sql) => wrapStatement(db.prepare(sql)), + run: async (sql) => { + db.run(sql); + }, + export: async () => db.export(), + close: async () => { + db.close(); + }, + }; +} + +export async function createSqlJsAdapter( + opts: SqlJsAdapterOptions = {} +): Promise { + const SQL = + opts.SQL || + (await initSqlJs(opts.locateFile ? { locateFile: opts.locateFile } : {})); + + return { + createEmpty: async () => wrapSqlJsDatabase(new SQL.Database()), + openFromBuffer: async (buf) => + wrapSqlJsDatabase( + new SQL.Database(buf instanceof Uint8Array ? buf : new Uint8Array(buf)) + ), + }; +} diff --git a/src/adapters/types.ts b/src/adapters/types.ts new file mode 100644 index 0000000..001042f --- /dev/null +++ b/src/adapters/types.ts @@ -0,0 +1,66 @@ +/** + * Adapter types for pluggable database backends. + * + * gtfs-sqljs defines a narrow, async surface that any SQLite driver can + * satisfy (sql.js, better-sqlite3, op-sqlite, expo-sqlite, …). The core + * library no longer imports sql.js directly; consumers pick an adapter and + * either pass a factory (`GtfsDatabaseAdapter`) into `fromZip` / `fromDatabase` + * or hand in a pre-opened `GtfsDatabase` via `GtfsSqlJs.attach()`. + */ + +export type SqlValue = string | number | null | Uint8Array; +export type Row = Record; + +export interface GtfsStatement { + /** Bind positional parameters. May be called once per execution. */ + bind(params: SqlValue[]): Promise; + /** Advance cursor. Resolves true if a row is available. */ + step(): Promise; + /** Materialize current row as an object keyed by column name. */ + getAsObject(): Promise; + /** Execute an INSERT/UPDATE/DELETE with parameters, no cursor needed. */ + run(params?: SqlValue[]): Promise; + /** Release native resources. Called after every prepare(). */ + free(): Promise; +} + +export interface GtfsDatabase { + /** Prepare a SQL statement for repeated execution. */ + prepare(sql: string): Promise; + /** Execute a one-shot SQL string (PRAGMA, DDL, transaction control). */ + run(sql: string): Promise; + /** + * Serialize the database to a byte buffer. + * File-backed drivers should throw `ExportNotSupportedError`. + */ + export(): Promise; + /** Close and release the database. */ + close(): Promise; +} + +/** + * Factory used by flows that need the library to create / load a DB on the + * caller's behalf (fromZip, fromDatabase). Not needed for `attach()`. + */ +export interface GtfsDatabaseAdapter { + /** Create an empty database (used by fromZip / fromZipData). */ + createEmpty(): Promise; + /** + * Open an existing database from a byte buffer (used by fromDatabase). + * File-backed drivers that cannot load bytes into a fresh DB may throw. + */ + openFromBuffer(buffer: ArrayBuffer | Uint8Array): Promise; +} + +/** + * Thrown by adapters whose underlying driver cannot serialize the database + * to an in-memory byte buffer (op-sqlite, expo-sqlite, better-sqlite3 against + * a file path). Callers that catch this can treat the database as + * non-exportable; the cache layer no-ops when it sees this error. + */ +export class ExportNotSupportedError extends Error { + constructor(message = 'export() is not supported by this adapter') { + super(message); + this.name = 'ExportNotSupportedError'; + } +} diff --git a/src/cache/checksum.ts b/src/cache/checksum.ts index 5f8c845..e5eba78 100644 --- a/src/cache/checksum.ts +++ b/src/cache/checksum.ts @@ -1,6 +1,28 @@ +interface WebCryptoLike { + subtle: { digest(algo: string, data: ArrayBuffer): Promise }; +} + +let cachedCrypto: WebCryptoLike | null = null; + +async function getCrypto(): Promise { + if (cachedCrypto) return cachedCrypto; + const g = (globalThis as unknown as { crypto?: WebCryptoLike }).crypto; + if (g && g.subtle) { + cachedCrypto = g; + return g; + } + // Node.js 18 does not expose `crypto` as a global; fall back to the + // `webcrypto` export of `node:crypto`. Browser/RN bundles never hit this + // branch because `globalThis.crypto` is always defined there. + const nodeCrypto = await import('node:crypto'); + cachedCrypto = nodeCrypto.webcrypto as unknown as WebCryptoLike; + return cachedCrypto; +} + /** * Compute SHA-256 checksum of data. - * Uses the global Web Crypto API (available in browsers and Node.js 18+). + * Uses the global Web Crypto API (browsers, Node.js 19+, React Native via a + * polyfill) and falls back to `node:crypto`'s webcrypto on Node 18. */ export async function computeChecksum(data: ArrayBuffer | Uint8Array): Promise { let buffer: ArrayBuffer; @@ -12,7 +34,8 @@ export async function computeChecksum(data: ArrayBuffer | Uint8Array): Promise b.toString(16).padStart(2, '0')).join(''); return hashHex; diff --git a/src/gtfs-sqljs.ts b/src/gtfs-sqljs.ts index 0128064..26fe309 100644 --- a/src/gtfs-sqljs.ts +++ b/src/gtfs-sqljs.ts @@ -2,7 +2,6 @@ * Main GtfsSqlJs Class */ -import initSqlJs, { type Database, type SqlJsStatic } from 'sql.js'; import { getAllCreateTableStatements, getAllCreateIndexStatements } from './schema/schema'; import { loadGTFSZip, fetchZip } from './loaders/zip-loader'; import { loadGTFSData } from './loaders/data-loader'; @@ -11,6 +10,8 @@ import { loadRealtimeData, loadRealtimeDataFromBuffers } from './loaders/gtfs-rt import type { CacheStore } from './cache/types'; import { computeZipChecksum, generateCacheKey } from './cache/checksum'; import { DEFAULT_CACHE_EXPIRATION_MS, isCacheExpired } from './cache/utils'; +import type { GtfsDatabase, GtfsDatabaseAdapter } from './adapters/types'; +import { ExportNotSupportedError } from './adapters/types'; // Library version from package.json const LIB_VERSION = '0.1.0'; @@ -85,14 +86,11 @@ export interface GtfsSqlJsOptions { database?: ArrayBuffer; /** - * Optional: Custom SQL.js instance + * Required: database adapter factory. Use `createSqlJsAdapter()` from + * `gtfs-sqljs/adapters/sql-js` for the browser/Node sql.js path, or plug + * in a custom adapter (op-sqlite, expo-sqlite, better-sqlite3, …). */ - SQL?: SqlJsStatic; - - /** - * Optional: Path to SQL.js WASM file (for custom loading) - */ - locateFile?: (filename: string) => string; + adapter: GtfsDatabaseAdapter; /** * Optional: Array of GTFS filenames to skip importing (e.g., ['shapes.txt']) @@ -142,9 +140,37 @@ export interface GtfsSqlJsOptions { cacheExpirationMs?: number; } +/** + * Options for `GtfsSqlJs.attach()` — when the caller already has a live DB handle. + */ +export interface GtfsSqlJsAttachOptions { + /** + * When `true`, assume the GTFS schema already exists in the attached DB + * and skip the `CREATE TABLE IF NOT EXISTS` DDL. Defaults to `false`: + * the library runs the idempotent schema DDL so that attaching to a fresh + * empty DB still works. + */ + skipSchema?: boolean; + + /** + * When `true`, `GtfsSqlJs.close()` will also close the underlying adapter + * handle. Defaults to `false` — callers retain ownership of handles they + * passed in via `attach()`. + */ + ownsDatabase?: boolean; + + /** Optional: Array of GTFS-RT feed URLs for realtime data */ + realtimeFeedUrls?: string[]; + + /** Optional: Staleness threshold in seconds (default: 120) */ + stalenessThreshold?: number; +} + +type FactoryOptions = Omit; + export class GtfsSqlJs { - private db: Database | null = null; - private SQL: SqlJsStatic | null = null; + private db: GtfsDatabase | null = null; + private ownsDatabase: boolean = true; private realtimeFeedUrls: string[] = []; private stalenessThreshold: number = 120; private lastRealtimeFetchTimestamp: number | null = null; @@ -159,8 +185,9 @@ export class GtfsSqlJs { */ static async fromZip( zipPath: string, - options: Omit = {} + options: FactoryOptions ): Promise { + assertAdapter(options, 'fromZip'); const zipData = await fetchZip(zipPath, options.onProgress); return GtfsSqlJs.fromZipData(zipData, options, zipPath); } @@ -171,9 +198,10 @@ export class GtfsSqlJs { */ static async fromZipData( zipData: ArrayBuffer | Uint8Array, - options: Omit = {}, + options: FactoryOptions, source?: string ): Promise { + assertAdapter(options, 'fromZipData'); const instance = new GtfsSqlJs(); await instance.initFromZipData(zipData, options, source); return instance; @@ -184,29 +212,66 @@ export class GtfsSqlJs { */ static async fromDatabase( database: ArrayBuffer, - options: Omit = {} + options: FactoryOptions ): Promise { + assertAdapter(options, 'fromDatabase'); const instance = new GtfsSqlJs(); await instance.initFromDatabase(database, options); return instance; } + /** + * Attach to a pre-opened database handle. + * + * Use this path when the caller already owns a live `GtfsDatabase` + * (typical for file-backed native drivers: op-sqlite, expo-sqlite, + * better-sqlite3). No adapter factory is needed — the handle *is* the + * adapter output. + */ + static async attach( + db: GtfsDatabase, + options: GtfsSqlJsAttachOptions = {} + ): Promise { + const instance = new GtfsSqlJs(); + instance.db = db; + instance.ownsDatabase = options.ownsDatabase === true; + + if (!options.skipSchema) { + const createTableStatements = getAllCreateTableStatements(); + for (const statement of createTableStatements) { + await db.run(statement); + } + await createRealtimeTables(db); + } + + if (options.realtimeFeedUrls) { + instance.realtimeFeedUrls = options.realtimeFeedUrls; + } + if (options.stalenessThreshold !== undefined) { + instance.stalenessThreshold = options.stalenessThreshold; + } + + return instance; + } + /** * Initialize from pre-loaded ZIP data * @param source - Optional original path/URL, used for cache key generation and metadata */ - private async initFromZipData(zipData: ArrayBuffer | Uint8Array, options: Omit, source?: string): Promise { + private async initFromZipData( + zipData: ArrayBuffer | Uint8Array, + options: FactoryOptions, + source?: string + ): Promise { const onProgress = options.onProgress; const { cache: userCache, cacheVersion = '1.0', cacheExpirationMs = DEFAULT_CACHE_EXPIRATION_MS, - skipFiles + skipFiles, + adapter, } = options; - // Initialize SQL.js - this.SQL = options.SQL || (await initSqlJs(options.locateFile ? { locateFile: options.locateFile } : {})); - // Determine cache store to use // Cache store must be provided explicitly by the user // See examples/cache/ for available implementations @@ -275,7 +340,8 @@ export class GtfsSqlJs { message: 'Loading from cache...', }); - this.db = new this.SQL.Database(new Uint8Array(cacheEntry.data)); + this.db = await adapter.openFromBuffer(new Uint8Array(cacheEntry.data)); + this.ownsDatabase = true; // Set RT configuration if (options.realtimeFeedUrls) { @@ -315,15 +381,26 @@ export class GtfsSqlJs { message: 'Saving to cache...', }); - const dbBuffer = this.export(); - await cache.set(cacheKey, dbBuffer, { - checksum, - version: cacheVersion, - timestamp: Date.now(), - source, - size: dbBuffer.byteLength, - skipFiles, - }); + try { + const dbBuffer = await this.export(); + await cache.set(cacheKey, dbBuffer, { + checksum, + version: cacheVersion, + timestamp: Date.now(), + source, + size: dbBuffer.byteLength, + skipFiles, + }); + } catch (error) { + if (error instanceof ExportNotSupportedError) { + console.warn( + 'Skipping cache write: the active adapter does not support export(). ' + + 'File-backed adapters persist their own DB on disk and do not need the library cache.' + ); + } else { + throw error; + } + } onProgress?.({ phase: 'complete', @@ -360,15 +437,12 @@ export class GtfsSqlJs { */ private async loadFromZipData( zipData: ArrayBuffer | Uint8Array, - options: Omit, + options: FactoryOptions, onProgress?: ProgressCallback ): Promise { - if (!this.SQL) { - throw new Error('SQL.js not initialized'); - } - - // Create new database - this.db = new this.SQL.Database(); + // Create new database via the adapter + this.db = await options.adapter.createEmpty(); + this.ownsDatabase = true; // Create GTFS tables (without indexes) onProgress?.({ @@ -384,11 +458,11 @@ export class GtfsSqlJs { const createTableStatements = getAllCreateTableStatements(); for (const statement of createTableStatements) { - this.db.run(statement); + await this.db.run(statement); } // Create GTFS-RT tables - createRealtimeTables(this.db); + await createRealtimeTables(this.db); // Extract files from zip onProgress?.({ @@ -433,7 +507,7 @@ export class GtfsSqlJs { const createIndexStatements = getAllCreateIndexStatements(); let indexCount = 0; for (const statement of createIndexStatements) { - this.db.run(statement); + await this.db.run(statement); indexCount++; const indexProgress = 75 + Math.floor((indexCount / createIndexStatements.length) * 10); onProgress?.({ @@ -460,7 +534,7 @@ export class GtfsSqlJs { message: 'Optimizing query performance', }); - this.db.run('ANALYZE'); + await this.db.run('ANALYZE'); // Set RT configuration if (options.realtimeFeedUrls) { @@ -497,16 +571,13 @@ export class GtfsSqlJs { */ private async initFromDatabase( database: ArrayBuffer, - options: Omit + options: FactoryOptions ): Promise { - // Initialize SQL.js - this.SQL = options.SQL || (await initSqlJs(options.locateFile ? { locateFile: options.locateFile } : {})); - - // Load existing database - this.db = new this.SQL.Database(new Uint8Array(database)); + this.db = await options.adapter.openFromBuffer(new Uint8Array(database)); + this.ownsDatabase = true; // Ensure RT tables exist (in case loading old database) - createRealtimeTables(this.db); + await createRealtimeTables(this.db); // Set RT configuration if (options.realtimeFeedUrls) { @@ -519,13 +590,16 @@ export class GtfsSqlJs { /** * Export database to ArrayBuffer + * + * Throws `ExportNotSupportedError` if the active adapter is file-backed + * and cannot serialize the database to bytes. */ - export(): ArrayBuffer { + async export(): Promise { if (!this.db) { throw new Error('Database not initialized'); } - const data = this.db.export(); + const data = await this.db.export(); // Create a new ArrayBuffer and copy the data to ensure proper type const buffer = new ArrayBuffer(data.length); new Uint8Array(buffer).set(data); @@ -533,19 +607,25 @@ export class GtfsSqlJs { } /** - * Close the database connection - */ - close(): void { - if (this.db) { - this.db.close(); - this.db = null; + * Close the database connection. + * + * When the library itself created the DB handle (via `fromZip`, + * `fromZipData`, `fromDatabase`, or when `attach()` was called with + * `ownsDatabase: true`), this also closes the underlying adapter. For + * handles passed to `attach()` without `ownsDatabase`, this is a no-op + * beyond clearing the internal reference — the caller owns the handle. + */ + async close(): Promise { + if (this.db && this.ownsDatabase) { + await this.db.close(); } + this.db = null; } /** - * Get direct access to the database (for advanced queries) + * Get direct access to the underlying adapter database handle (for advanced queries). */ - getDatabase(): Database { + getDatabase(): GtfsDatabase { if (!this.db) { throw new Error('Database not initialized'); } @@ -558,7 +638,7 @@ export class GtfsSqlJs { * Get agencies with optional filters * Pass agencyId filter to get a specific agency */ - getAgencies(filters?: AgencyFilters): Agency[] { + async getAgencies(filters?: AgencyFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getAgencies(this.db, filters); } @@ -569,7 +649,7 @@ export class GtfsSqlJs { * Get stops with optional filters * Pass stopId filter to get a specific stop */ - getStops(filters?: StopFilters): Stop[] { + async getStops(filters?: StopFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getStops(this.db, filters); } @@ -580,7 +660,7 @@ export class GtfsSqlJs { * Get routes with optional filters * Pass routeId filter to get a specific route */ - getRoutes(filters?: RouteFilters): Route[] { + async getRoutes(filters?: RouteFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getRoutes(this.db, filters); } @@ -590,7 +670,7 @@ export class GtfsSqlJs { /** * Get active service IDs for a given date (YYYYMMDD format) */ - getActiveServiceIds(date: string): string[] { + async getActiveServiceIds(date: string): Promise { if (!this.db) throw new Error('Database not initialized'); return getActiveServiceIds(this.db, date); } @@ -598,7 +678,7 @@ export class GtfsSqlJs { /** * Get calendar entry by service_id */ - getCalendarByServiceId(serviceId: string): Calendar | null { + async getCalendarByServiceId(serviceId: string): Promise { if (!this.db) throw new Error('Database not initialized'); return getCalendarByServiceId(this.db, serviceId); } @@ -606,7 +686,7 @@ export class GtfsSqlJs { /** * Get calendar date exceptions for a service */ - getCalendarDates(serviceId: string): CalendarDate[] { + async getCalendarDates(serviceId: string): Promise { if (!this.db) throw new Error('Database not initialized'); return getCalendarDates(this.db, serviceId); } @@ -614,7 +694,7 @@ export class GtfsSqlJs { /** * Get calendar date exceptions for a specific date */ - getCalendarDatesForDate(date: string): CalendarDate[] { + async getCalendarDatesForDate(date: string): Promise { if (!this.db) throw new Error('Database not initialized'); return getCalendarDatesForDate(this.db, date); } @@ -624,28 +704,8 @@ export class GtfsSqlJs { /** * Get trips with optional filters * Pass tripId filter to get a specific trip - * - * @param filters - Optional filters - * @param filters.tripId - Filter by trip ID (single value or array) - * @param filters.routeId - Filter by route ID (single value or array) - * @param filters.date - Filter by date (YYYYMMDD format) - will get active services for that date - * @param filters.directionId - Filter by direction ID (single value or array) - * @param filters.agencyId - Filter by agency ID (single value or array) - * @param filters.limit - Limit number of results - * - * @example - * // Get all trips for a route on a specific date - * const trips = gtfs.getTrips({ routeId: 'ROUTE_1', date: '20240115' }); - * - * @example - * // Get all trips for a route going in one direction - * const trips = gtfs.getTrips({ routeId: 'ROUTE_1', directionId: 0 }); - * - * @example - * // Get a specific trip - * const trips = gtfs.getTrips({ tripId: 'TRIP_123' }); */ - getTrips(filters?: TripFilters & { date?: string }): Trip[] { + async getTrips(filters?: TripFilters & { date?: string }): Promise { if (!this.db) throw new Error('Database not initialized'); // Handle date parameter by converting it to serviceIds @@ -653,7 +713,7 @@ export class GtfsSqlJs { const finalFilters = { ...restFilters }; if (date) { - const serviceIds = getActiveServiceIds(this.db, date); + const serviceIds = await getActiveServiceIds(this.db, date); finalFilters.serviceIds = serviceIds; } @@ -664,75 +724,16 @@ export class GtfsSqlJs { /** * Get shapes with optional filters - * - * @param filters - Optional filters - * @param filters.shapeId - Filter by shape ID (single value or array) - * @param filters.routeId - Filter by route ID (single value or array) - joins with trips table - * @param filters.tripId - Filter by trip ID (single value or array) - joins with trips table - * @param filters.limit - Limit number of results - * - * @example - * // Get all points for a specific shape - * const shapes = gtfs.getShapes({ shapeId: 'SHAPE_1' }); - * - * @example - * // Get shapes for a specific route - * const shapes = gtfs.getShapes({ routeId: 'ROUTE_1' }); - * - * @example - * // Get shapes for multiple trips - * const shapes = gtfs.getShapes({ tripId: ['TRIP_1', 'TRIP_2'] }); */ - getShapes(filters?: ShapeFilters): Shape[] { + async getShapes(filters?: ShapeFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getShapes(this.db, filters); } /** * Get shapes as GeoJSON FeatureCollection - * - * Each shape is converted to a LineString Feature with route properties. - * Coordinates are in [longitude, latitude] format per GeoJSON spec. - * - * @param filters - Optional filters (same as getShapes) - * @param filters.shapeId - Filter by shape ID (single value or array) - * @param filters.routeId - Filter by route ID (single value or array) - * @param filters.tripId - Filter by trip ID (single value or array) - * @param filters.limit - Limit number of results - * @param precision - Number of decimal places for coordinates (default: 6, ~10cm precision) - * - * @returns GeoJSON FeatureCollection with LineString features - * - * @example - * // Get all shapes as GeoJSON - * const geojson = gtfs.getShapesToGeojson(); - * - * @example - * // Get shapes for a route with lower precision - * const geojson = gtfs.getShapesToGeojson({ routeId: 'ROUTE_1' }, 5); - * - * @example - * // Result structure: - * // { - * // type: 'FeatureCollection', - * // features: [{ - * // type: 'Feature', - * // properties: { - * // shape_id: 'SHAPE_1', - * // route_id: 'ROUTE_1', - * // route_short_name: '1', - * // route_long_name: 'Main Street', - * // route_type: 3, - * // route_color: 'FF0000' - * // }, - * // geometry: { - * // type: 'LineString', - * // coordinates: [[-122.123456, 37.123456], ...] - * // } - * // }] - * // } */ - getShapesToGeojson(filters?: ShapeFilters, precision: number = 6): GeoJsonFeatureCollection { + async getShapesToGeojson(filters?: ShapeFilters, precision: number = 6): Promise { if (!this.db) throw new Error('Database not initialized'); return getShapesToGeojson(this.db, filters, precision); } @@ -741,37 +742,8 @@ export class GtfsSqlJs { /** * Get stop times with optional filters - * - * @param filters - Optional filters - * @param filters.tripId - Filter by trip ID (single value or array) - * @param filters.stopId - Filter by stop ID (single value or array) - * @param filters.routeId - Filter by route ID (single value or array) - * @param filters.date - Filter by date (YYYYMMDD format) - will get active services for that date - * @param filters.directionId - Filter by direction ID (single value or array) - * @param filters.agencyId - Filter by agency ID (single value or array) - * @param filters.includeRealtime - Include realtime data (delay and time fields) - * @param filters.limit - Limit number of results - * - * @example - * // Get stop times for a specific trip - * const stopTimes = gtfs.getStopTimes({ tripId: 'TRIP_123' }); - * - * @example - * // Get stop times at a stop for a specific route on a date - * const stopTimes = gtfs.getStopTimes({ - * stopId: 'STOP_123', - * routeId: 'ROUTE_1', - * date: '20240115' - * }); - * - * @example - * // Get stop times with realtime data - * const stopTimes = gtfs.getStopTimes({ - * tripId: 'TRIP_123', - * includeRealtime: true - * }); */ - getStopTimes(filters?: StopTimeFilters & { date?: string }): StopTime[] { + async getStopTimes(filters?: StopTimeFilters & { date?: string }): Promise { if (!this.db) throw new Error('Database not initialized'); // Handle date parameter by converting it to serviceIds @@ -779,7 +751,7 @@ export class GtfsSqlJs { const finalFilters = { ...restFilters }; if (date) { - const serviceIds = getActiveServiceIds(this.db, date); + const serviceIds = await getActiveServiceIds(this.db, date); finalFilters.serviceIds = serviceIds; } @@ -788,31 +760,8 @@ export class GtfsSqlJs { /** * Build an ordered list of stops from multiple trips - * - * This is useful when you need to display a timetable for a route where different trips - * may stop at different sets of stops (e.g., express vs local service, or trips with - * different start/end points). - * - * The method intelligently merges stop sequences from all provided trips to create - * a comprehensive ordered list of all unique stops. - * - * @param tripIds - Array of trip IDs to analyze - * @returns Ordered array of Stop objects representing all unique stops - * - * @example - * // Get all trips for a route going in one direction - * const trips = gtfs.getTrips({ routeId: 'ROUTE_1', directionId: 0 }); - * const tripIds = trips.map(t => t.trip_id); - * - * // Build ordered stop list for all these trips - * const stops = gtfs.buildOrderedStopList(tripIds); - * - * // Now you can display a timetable with all possible stops - * stops.forEach(stop => { - * console.log(stop.stop_name); - * }); */ - buildOrderedStopList(tripIds: string[]): Stop[] { + async buildOrderedStopList(tripIds: string[]): Promise { if (!this.db) throw new Error('Database not initialized'); return buildOrderedStopList(this.db, tripIds); } @@ -849,7 +798,6 @@ export class GtfsSqlJs { /** * Get timestamp of the last successful realtime data fetch and insertion - * @returns Unix timestamp in seconds, or null if no realtime data has been fetched */ getLastRealtimeFetchTimestamp(): number | null { return this.lastRealtimeFetchTimestamp; @@ -857,7 +805,6 @@ export class GtfsSqlJs { /** * Fetch and load GTFS Realtime data from configured feed URLs or provided URLs - * @param urls - Optional array of feed URLs. If not provided, uses configured feed URLs */ async fetchRealtimeData(urls?: string[]): Promise { if (!this.db) throw new Error('Database not initialized'); @@ -873,7 +820,6 @@ export class GtfsSqlJs { /** * Load GTFS Realtime data from pre-loaded protobuf buffers - * @param buffers - Array of Uint8Array protobuf-encoded GTFS-RT feed messages */ async loadRealtimeDataFromBuffers(buffers: Uint8Array[]): Promise { if (!this.db) throw new Error('Database not initialized'); @@ -885,91 +831,67 @@ export class GtfsSqlJs { /** * Clear all realtime data from the database */ - clearRealtimeData(): void { + async clearRealtimeData(): Promise { if (!this.db) throw new Error('Database not initialized'); - clearRTData(this.db); + await clearRTData(this.db); } /** * Get alerts with optional filters - * Pass alertId filter to get a specific alert */ - getAlerts(filters?: AlertFilters): Alert[] { + async getAlerts(filters?: AlertFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getAlertsQuery(this.db, filters, this.stalenessThreshold); } /** * Get vehicle positions with optional filters - * Pass tripId filter to get vehicle position for a specific trip */ - getVehiclePositions(filters?: VehiclePositionFilters): VehiclePosition[] { + async getVehiclePositions(filters?: VehiclePositionFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getVehiclePositionsQuery(this.db, filters, this.stalenessThreshold); } /** * Get trip updates with optional filters - * Pass tripId filter to get trip update for a specific trip */ - getTripUpdates(filters?: TripUpdateFilters): TripUpdate[] { + async getTripUpdates(filters?: TripUpdateFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getTripUpdates(this.db, filters, this.stalenessThreshold); } /** * Get stop time updates with optional filters - * Pass tripId filter to get stop time updates for a specific trip */ - getStopTimeUpdates(filters?: StopTimeUpdateFilters): import('./types/gtfs-rt').StopTimeUpdate[] { + async getStopTimeUpdates(filters?: StopTimeUpdateFilters): Promise { if (!this.db) throw new Error('Database not initialized'); return getStopTimeUpdates(this.db, filters, this.stalenessThreshold); } // ==================== Debug Export Methods ==================== - // These methods export all realtime data without staleness filtering - // for debugging purposes - /** - * Export all alerts without staleness filtering (for debugging) - */ - debugExportAllAlerts(): Alert[] { + async debugExportAllAlerts(): Promise { if (!this.db) throw new Error('Database not initialized'); return getAllAlerts(this.db); } - /** - * Export all vehicle positions without staleness filtering (for debugging) - */ - debugExportAllVehiclePositions(): VehiclePosition[] { + async debugExportAllVehiclePositions(): Promise { if (!this.db) throw new Error('Database not initialized'); return getAllVehiclePositions(this.db); } - /** - * Export all trip updates without staleness filtering (for debugging) - */ - debugExportAllTripUpdates(): TripUpdate[] { + async debugExportAllTripUpdates(): Promise { if (!this.db) throw new Error('Database not initialized'); return getAllTripUpdates(this.db); } - /** - * Export all stop time updates without staleness filtering (for debugging) - * Returns stop time updates with trip_id and rt_last_updated populated - */ - debugExportAllStopTimeUpdates(): StopTimeUpdate[] { + async debugExportAllStopTimeUpdates(): Promise { if (!this.db) throw new Error('Database not initialized'); return getAllStopTimeUpdates(this.db); } // ==================== Cache Management Methods ==================== - /** - * Get cache statistics - * @param cacheStore - Cache store to query (required) - * @returns Cache statistics including size, entry count, and age information - */ static async getCacheStats(cacheStore: CacheStore) { const { getCacheStats } = await import('./cache/utils'); @@ -981,12 +903,6 @@ export class GtfsSqlJs { return getCacheStats(entries); } - /** - * Clean expired cache entries - * @param cacheStore - Cache store to clean (required) - * @param expirationMs - Expiration time in milliseconds (default: 7 days) - * @returns Number of entries deleted - */ static async cleanExpiredCache( cacheStore: CacheStore, expirationMs: number = DEFAULT_CACHE_EXPIRATION_MS @@ -1002,16 +918,11 @@ export class GtfsSqlJs { !filterExpiredEntries([entry], expirationMs).length ); - // Delete expired entries await Promise.all(expiredEntries.map(entry => cacheStore.delete(entry.key))); return expiredEntries.length; } - /** - * Clear all cache entries - * @param cacheStore - Cache store to clear (required) - */ static async clearCache(cacheStore: CacheStore): Promise { if (!cacheStore) { throw new Error('Cache store is required'); @@ -1020,12 +931,6 @@ export class GtfsSqlJs { await cacheStore.clear(); } - /** - * List all cache entries - * @param cacheStore - Cache store to query (required) - * @param includeExpired - Include expired entries (default: false) - * @returns Array of cache entries with metadata - */ static async listCache( cacheStore: CacheStore, includeExpired: boolean = false @@ -1045,3 +950,14 @@ export class GtfsSqlJs { return filterExpiredEntries(entries); } } + +function assertAdapter(options: FactoryOptions | undefined, method: string): void { + if (!options || !options.adapter) { + throw new Error( + `${method}() requires an \`adapter\`. Pass one via options — e.g. ` + + `import { createSqlJsAdapter } from 'gtfs-sqljs/adapters/sql-js' and ` + + `set \`adapter: await createSqlJsAdapter()\`, or call GtfsSqlJs.attach() ` + + `with an already-open handle.` + ); + } +} diff --git a/src/index.ts b/src/index.ts index 3b5aaab..698677e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ /** - * gtfs-sqljs - Load GTFS data into sql.js SQLite database + * gtfs-sqljs - Load GTFS data into a SQLite database (pluggable adapter) * @author Théophile Helleboid/SysDevRun * @license MIT */ @@ -7,6 +7,7 @@ export { GtfsSqlJs, type GtfsSqlJsOptions, + type GtfsSqlJsAttachOptions, type AgencyFilters, type StopFilters, type RouteFilters, @@ -25,6 +26,16 @@ export { type GeoJsonFeatureCollection } from './gtfs-sqljs'; +// Export adapter surface (types + error class) +export type { + SqlValue, + Row, + GtfsStatement, + GtfsDatabase, + GtfsDatabaseAdapter, +} from './adapters/types'; +export { ExportNotSupportedError } from './adapters/types'; + // Export GTFS types export type { Agency, @@ -90,3 +101,6 @@ export { isCacheExpired, filterExpiredEntries, getCacheStats, DEFAULT_CACHE_EXPI // Note: Cache store implementations (IndexedDBCacheStore, FileSystemCacheStore) are available // in examples/cache/ directory. Copy them to your project as needed. +// Note: Database adapters live at subpaths: +// - `gtfs-sqljs/adapters/sql-js` (browser / Node WASM path) +// - `gtfs-sqljs/adapters/better-sqlite3` (Node file-backed / in-memory path) diff --git a/src/loaders/data-loader.ts b/src/loaders/data-loader.ts index 9a8bbb1..0429c01 100644 --- a/src/loaders/data-loader.ts +++ b/src/loaders/data-loader.ts @@ -2,12 +2,12 @@ * Data Loader - Loads GTFS data into SQLite database */ -import type { Database } from 'sql.js'; import Papa from 'papaparse'; import { countCsvRows } from './csv-parser'; import { GTFS_SCHEMA, type TableSchema } from '../schema/schema'; import type { GTFSFiles } from './zip-loader'; import type { ProgressCallback } from '../gtfs-sqljs'; +import type { GtfsDatabase, SqlValue } from '../adapters/types'; const PROGRESS_BATCH = 1000; @@ -17,7 +17,7 @@ const PROGRESS_BATCH = 1000; * @param onProgress - Optional progress callback */ export async function loadGTFSData( - db: Database, + db: GtfsDatabase, files: GTFSFiles, skipFiles?: string[], onProgress?: ProgressCallback @@ -144,7 +144,7 @@ function computePercent(rowsProcessed: number, totalRows: number): number { * allocation) and binds column values by pre-computed index. */ async function loadTableData( - db: Database, + db: GtfsDatabase, schema: TableSchema, csvContent: string, onProgress?: (rowsProcessed: number) => void @@ -174,11 +174,11 @@ async function loadTableData( .map(() => '?') .join(', ')})`; - db.run('BEGIN TRANSACTION'); + await db.run('BEGIN TRANSACTION'); try { - const stmt = db.prepare(insertSQL); + const stmt = await db.prepare(insertSQL); try { - const rowVals: (string | number | null)[] = new Array(columns.length); + const rowVals: SqlValue[] = new Array(columns.length); for (let r = 0; r < dataRows.length; r++) { const row = dataRows[r]; for (let j = 0; j < colIndexes.length; j++) { @@ -190,19 +190,19 @@ async function loadTableData( rowVals[j] = trimmed === '' ? null : trimmed; } } - stmt.run(rowVals); + await stmt.run(rowVals); const done = r + 1; if (done % PROGRESS_BATCH === 0) onProgress?.(done); } if (dataRows.length % PROGRESS_BATCH !== 0) onProgress?.(dataRows.length); } finally { - stmt.free(); + await stmt.free(); } - db.run('COMMIT'); + await db.run('COMMIT'); } catch (error) { try { - db.run('ROLLBACK'); + await db.run('ROLLBACK'); } catch (rollbackError) { console.error('Error rolling back transaction:', rollbackError); } diff --git a/src/loaders/gtfs-rt-loader.ts b/src/loaders/gtfs-rt-loader.ts index c3af066..c1395ef 100644 --- a/src/loaders/gtfs-rt-loader.ts +++ b/src/loaders/gtfs-rt-loader.ts @@ -1,4 +1,4 @@ -import type { Database } from 'sql.js'; +import type { GtfsDatabase } from '../adapters/types'; import protobuf from 'protobufjs'; @@ -348,8 +348,8 @@ function parseTranslatedString(ts: ProtobufTranslatedString | undefined): string } // Insert alerts into database -function insertAlerts(db: Database, alerts: ProtobufAlert[], timestamp: number): void { - const stmt = db.prepare(` +async function insertAlerts(db: GtfsDatabase, alerts: ProtobufAlert[], timestamp: number): Promise { + const stmt = await db.prepare(` INSERT OR REPLACE INTO rt_alerts ( id, active_period, informed_entity, cause, effect, url, header_text, description_text, rt_last_updated @@ -361,7 +361,7 @@ function insertAlerts(db: Database, alerts: ProtobufAlert[], timestamp: number): const activePeriodSnake = (alert.activePeriod ?? []).map(convertObjectKeysToSnakeCase); const informedEntitySnake = (alert.informedEntity ?? []).map(convertObjectKeysToSnakeCase); - stmt.run([ + await stmt.run([ alert.id, JSON.stringify(activePeriodSnake), JSON.stringify(informedEntitySnake), @@ -374,12 +374,12 @@ function insertAlerts(db: Database, alerts: ProtobufAlert[], timestamp: number): ]); } - stmt.free(); + await stmt.free(); } // Insert vehicle positions into database -function insertVehiclePositions(db: Database, positions: ProtobufVehiclePosition[], timestamp: number): void { - const stmt = db.prepare(` +async function insertVehiclePositions(db: GtfsDatabase, positions: ProtobufVehiclePosition[], timestamp: number): Promise { + const stmt = await db.prepare(` INSERT OR REPLACE INTO rt_vehicle_positions ( trip_id, route_id, vehicle_id, vehicle_label, vehicle_license_plate, latitude, longitude, bearing, odometer, speed, @@ -392,7 +392,7 @@ function insertVehiclePositions(db: Database, positions: ProtobufVehiclePosition const trip = vp.trip; if (!trip || !trip.tripId) continue; - stmt.run([ + await stmt.run([ trip.tripId, trip.routeId || null, vp.vehicle?.id || null, @@ -413,19 +413,19 @@ function insertVehiclePositions(db: Database, positions: ProtobufVehiclePosition ]); } - stmt.free(); + await stmt.free(); } // Insert trip updates into database -function insertTripUpdates(db: Database, updates: ProtobufTripUpdate[], timestamp: number): void { - const tripStmt = db.prepare(` +async function insertTripUpdates(db: GtfsDatabase, updates: ProtobufTripUpdate[], timestamp: number): Promise { + const tripStmt = await db.prepare(` INSERT OR REPLACE INTO rt_trip_updates ( trip_id, route_id, vehicle_id, vehicle_label, vehicle_license_plate, timestamp, delay, schedule_relationship, rt_last_updated ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) `); - const stopTimeStmt = db.prepare(` + const stopTimeStmt = await db.prepare(` INSERT OR REPLACE INTO rt_stop_time_updates ( trip_id, stop_sequence, stop_id, arrival_delay, arrival_time, arrival_uncertainty, @@ -439,7 +439,7 @@ function insertTripUpdates(db: Database, updates: ProtobufTripUpdate[], timestam if (!trip || !trip.tripId) continue; // Insert trip update - tripStmt.run([ + await tripStmt.run([ trip.tripId, trip.routeId || null, tu.vehicle?.id || null, @@ -454,7 +454,7 @@ function insertTripUpdates(db: Database, updates: ProtobufTripUpdate[], timestam // Insert stop time updates if (tu.stopTimeUpdate) { for (const stu of tu.stopTimeUpdate) { - stopTimeStmt.run([ + await stopTimeStmt.run([ trip.tripId, stu.stopSequence || null, stu.stopId || null, @@ -471,8 +471,8 @@ function insertTripUpdates(db: Database, updates: ProtobufTripUpdate[], timestam } } - tripStmt.free(); - stopTimeStmt.free(); + await tripStmt.free(); + await stopTimeStmt.free(); } // Decode protobuf data into feed objects @@ -492,7 +492,7 @@ function decodeFeedMessage(data: Uint8Array): object { // Process decoded feed objects: collect entities and insert into database // eslint-disable-next-line @typescript-eslint/no-explicit-any -function processRealtimeFeeds(db: Database, feeds: any[]): void { +async function processRealtimeFeeds(db: GtfsDatabase, feeds: any[]): Promise { // Current timestamp for staleness tracking const now = Math.floor(Date.now() / 1000); @@ -518,28 +518,28 @@ function processRealtimeFeeds(db: Database, feeds: any[]): void { } // Clear old data and insert new data in a single transaction - db.run('BEGIN TRANSACTION'); + await db.run('BEGIN TRANSACTION'); try { // Clear all realtime tables - db.run('DELETE FROM rt_stop_time_updates'); - db.run('DELETE FROM rt_trip_updates'); - db.run('DELETE FROM rt_vehicle_positions'); - db.run('DELETE FROM rt_alerts'); + await db.run('DELETE FROM rt_stop_time_updates'); + await db.run('DELETE FROM rt_trip_updates'); + await db.run('DELETE FROM rt_vehicle_positions'); + await db.run('DELETE FROM rt_alerts'); // Insert new data if (allAlerts.length > 0) { - insertAlerts(db, allAlerts, now); + await insertAlerts(db, allAlerts, now); } if (allVehiclePositions.length > 0) { - insertVehiclePositions(db, allVehiclePositions, now); + await insertVehiclePositions(db, allVehiclePositions, now); } if (allTripUpdates.length > 0) { - insertTripUpdates(db, allTripUpdates, now); + await insertTripUpdates(db, allTripUpdates, now); } - db.run('COMMIT'); + await db.run('COMMIT'); } catch (error) { - db.run('ROLLBACK'); + await db.run('ROLLBACK'); throw error; } } @@ -548,7 +548,7 @@ function processRealtimeFeeds(db: Database, feeds: any[]): void { * Fetch and load GTFS Realtime data from multiple feed URLs * All feeds are fetched in parallel. If any feed fails, an error is thrown. */ -export async function loadRealtimeData(db: Database, feedUrls: string[]): Promise { +export async function loadRealtimeData(db: GtfsDatabase, feedUrls: string[]): Promise { // Fetch all feeds in parallel const buffers = await Promise.all(feedUrls.map(async (url) => { try { @@ -565,7 +565,7 @@ export async function loadRealtimeData(db: Database, feedUrls: string[]): Promis * Load GTFS Realtime data from pre-loaded protobuf buffers * Decodes and inserts data from Uint8Array buffers — same as loadRealtimeData but skips fetching. */ -export async function loadRealtimeDataFromBuffers(db: Database, buffers: Uint8Array[]): Promise { +export async function loadRealtimeDataFromBuffers(db: GtfsDatabase, buffers: Uint8Array[]): Promise { const feeds = buffers.map((buffer) => decodeFeedMessage(buffer)); - processRealtimeFeeds(db, feeds); + await processRealtimeFeeds(db, feeds); } diff --git a/src/queries/agencies.ts b/src/queries/agencies.ts index cdb2491..5c2bd2a 100644 --- a/src/queries/agencies.ts +++ b/src/queries/agencies.ts @@ -2,7 +2,7 @@ * Agency Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Agency } from '../types/gtfs'; export interface AgencyFilters { @@ -14,7 +14,7 @@ export interface AgencyFilters { * Get agencies with optional filters * - Filters support both single values and arrays */ -export function getAgencies(db: Database, filters: AgencyFilters = {}): Agency[] { +export async function getAgencies(db: GtfsDatabase, filters: AgencyFilters = {}): Promise { const { agencyId, limit } = filters; // Build WHERE clause dynamically @@ -41,25 +41,25 @@ export function getAgencies(db: Database, filters: AgencyFilters = {}): Agency[] params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const agencies: Agency[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); agencies.push(rowToAgency(row)); } - stmt.free(); + await stmt.free(); return agencies; } /** * Convert database row to Agency object */ -function rowToAgency(row: ParamsObject): Agency { +function rowToAgency(row: Row): Agency { return { agency_id: String(row.agency_id), agency_name: String(row.agency_name), diff --git a/src/queries/calendar.ts b/src/queries/calendar.ts index 9727ca0..12ef793 100644 --- a/src/queries/calendar.ts +++ b/src/queries/calendar.ts @@ -2,13 +2,13 @@ * Calendar Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Calendar, CalendarDate } from '../types/gtfs'; /** * Get active service IDs for a given date */ -export function getActiveServiceIds(db: Database, date: string): string[] { +export async function getActiveServiceIds(db: GtfsDatabase, date: string): Promise { const serviceIds = new Set(); // Parse date (format: YYYYMMDD) @@ -23,26 +23,26 @@ export function getActiveServiceIds(db: Database, date: string): string[] { const dayField = dayFields[dayOfWeek]; // Check calendar.txt for regular service - const calendarStmt = db.prepare( + const calendarStmt = await db.prepare( `SELECT service_id FROM calendar WHERE ${dayField} = 1 AND start_date <= ? AND end_date >= ?` ); - calendarStmt.bind([date, date]); + await calendarStmt.bind([date, date]); - while (calendarStmt.step()) { - const row = calendarStmt.getAsObject() as { service_id: string }; + while (await calendarStmt.step()) { + const row = await calendarStmt.getAsObject() as { service_id: string }; serviceIds.add(row.service_id); } - calendarStmt.free(); + await calendarStmt.free(); // Check calendar_dates.txt for exceptions - const exceptionsStmt = db.prepare('SELECT service_id, exception_type FROM calendar_dates WHERE date = ?'); - exceptionsStmt.bind([date]); + const exceptionsStmt = await db.prepare('SELECT service_id, exception_type FROM calendar_dates WHERE date = ?'); + await exceptionsStmt.bind([date]); - while (exceptionsStmt.step()) { - const row = exceptionsStmt.getAsObject() as { service_id: string; exception_type: number }; + while (await exceptionsStmt.step()) { + const row = await exceptionsStmt.getAsObject() as { service_id: string; exception_type: number }; if (row.exception_type === 1) { // Service added serviceIds.add(row.service_id); @@ -51,7 +51,7 @@ export function getActiveServiceIds(db: Database, date: string): string[] { serviceIds.delete(row.service_id); } } - exceptionsStmt.free(); + await exceptionsStmt.free(); return Array.from(serviceIds); } @@ -59,58 +59,58 @@ export function getActiveServiceIds(db: Database, date: string): string[] { /** * Get calendar entry by service_id */ -export function getCalendarByServiceId(db: Database, serviceId: string): Calendar | null { - const stmt = db.prepare('SELECT * FROM calendar WHERE service_id = ?'); - stmt.bind([serviceId]); +export async function getCalendarByServiceId(db: GtfsDatabase, serviceId: string): Promise { + const stmt = await db.prepare('SELECT * FROM calendar WHERE service_id = ?'); + await stmt.bind([serviceId]); - if (stmt.step()) { - const row = stmt.getAsObject(); - stmt.free(); + if (await stmt.step()) { + const row = await stmt.getAsObject(); + await stmt.free(); return rowToCalendar(row); } - stmt.free(); + await stmt.free(); return null; } /** * Get calendar date exceptions for a service */ -export function getCalendarDates(db: Database, serviceId: string): CalendarDate[] { - const stmt = db.prepare('SELECT * FROM calendar_dates WHERE service_id = ? ORDER BY date'); - stmt.bind([serviceId]); +export async function getCalendarDates(db: GtfsDatabase, serviceId: string): Promise { + const stmt = await db.prepare('SELECT * FROM calendar_dates WHERE service_id = ? ORDER BY date'); + await stmt.bind([serviceId]); const dates: CalendarDate[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); dates.push(rowToCalendarDate(row)); } - stmt.free(); + await stmt.free(); return dates; } /** * Get calendar date exceptions for a specific date */ -export function getCalendarDatesForDate(db: Database, date: string): CalendarDate[] { - const stmt = db.prepare('SELECT * FROM calendar_dates WHERE date = ?'); - stmt.bind([date]); +export async function getCalendarDatesForDate(db: GtfsDatabase, date: string): Promise { + const stmt = await db.prepare('SELECT * FROM calendar_dates WHERE date = ?'); + await stmt.bind([date]); const dates: CalendarDate[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); dates.push(rowToCalendarDate(row)); } - stmt.free(); + await stmt.free(); return dates; } /** * Convert database row to Calendar object */ -function rowToCalendar(row: ParamsObject): Calendar { +function rowToCalendar(row: Row): Calendar { return { service_id: String(row.service_id), monday: Number(row.monday), @@ -128,7 +128,7 @@ function rowToCalendar(row: ParamsObject): Calendar { /** * Convert database row to CalendarDate object */ -function rowToCalendarDate(row: ParamsObject): CalendarDate { +function rowToCalendarDate(row: Row): CalendarDate { return { service_id: String(row.service_id), date: String(row.date), diff --git a/src/queries/routes.ts b/src/queries/routes.ts index 6855303..b2e385f 100644 --- a/src/queries/routes.ts +++ b/src/queries/routes.ts @@ -2,7 +2,7 @@ * Route Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Route } from '../types/gtfs'; export interface RouteFilters { @@ -15,7 +15,7 @@ export interface RouteFilters { * Get routes with optional filters * - Filters support both single values and arrays */ -export function getRoutes(db: Database, filters: RouteFilters = {}): Route[] { +export async function getRoutes(db: GtfsDatabase, filters: RouteFilters = {}): Promise { const { routeId, agencyId, limit } = filters; // Build WHERE clause dynamically @@ -51,25 +51,25 @@ export function getRoutes(db: Database, filters: RouteFilters = {}): Route[] { params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const routes: Route[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); routes.push(rowToRoute(row)); } - stmt.free(); + await stmt.free(); return routes; } /** * Convert database row to Route object */ -function rowToRoute(row: ParamsObject): Route { +function rowToRoute(row: Row): Route { return { route_id: String(row.route_id), route_short_name: row.route_short_name ? String(row.route_short_name) : undefined, diff --git a/src/queries/rt-alerts.ts b/src/queries/rt-alerts.ts index 3b3ea04..dac3ef9 100644 --- a/src/queries/rt-alerts.ts +++ b/src/queries/rt-alerts.ts @@ -1,4 +1,4 @@ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Alert, EntitySelector, TimeRange, TranslatedString } from '../types/gtfs-rt'; import type { AlertFilters } from '../types/gtfs-rt'; @@ -7,7 +7,7 @@ export type { AlertFilters }; /** * Parse JSON fields from database */ -function parseAlert(row: ParamsObject): Alert { +function parseAlert(row: Row): Alert { return { id: String(row.id), active_period: row.active_period ? JSON.parse(String(row.active_period)) as TimeRange[] : [], @@ -74,7 +74,7 @@ function alertAffectsEntity(alert: Alert, filters: AlertFilters): boolean { /** * Get alerts with optional filters */ -export function getAlerts(db: Database, filters: AlertFilters = {}, stalenessThreshold: number = 120): Alert[] { +export async function getAlerts(db: GtfsDatabase, filters: AlertFilters = {}, stalenessThreshold: number = 120): Promise { const { alertId, activeOnly, @@ -126,14 +126,14 @@ export function getAlerts(db: Database, filters: AlertFilters = {}, stalenessThr } // Execute query - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const alerts: Alert[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); const alert = parseAlert(row); // Apply activeOnly filter in application code @@ -151,31 +151,31 @@ export function getAlerts(db: Database, filters: AlertFilters = {}, stalenessThr alerts.push(alert); } - stmt.free(); + await stmt.free(); return alerts; } /** * Get alert by ID */ -export function getAlertById(db: Database, alertId: string, stalenessThreshold: number = 120): Alert | null { - const alerts = getAlerts(db, { alertId, limit: 1 }, stalenessThreshold); +export async function getAlertById(db: GtfsDatabase, alertId: string, stalenessThreshold: number = 120): Promise { + const alerts = await getAlerts(db, { alertId, limit: 1 }, stalenessThreshold); return alerts.length > 0 ? alerts[0] : null; } /** * Get all alerts without staleness filtering (for debugging) */ -export function getAllAlerts(db: Database): Alert[] { +export async function getAllAlerts(db: GtfsDatabase): Promise { const sql = 'SELECT * FROM rt_alerts ORDER BY rt_last_updated DESC'; - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); const alerts: Alert[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); alerts.push(parseAlert(row)); } - stmt.free(); + await stmt.free(); return alerts; } diff --git a/src/queries/rt-stop-time-updates.ts b/src/queries/rt-stop-time-updates.ts index 2c11dce..da6da8b 100644 --- a/src/queries/rt-stop-time-updates.ts +++ b/src/queries/rt-stop-time-updates.ts @@ -1,4 +1,4 @@ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { StopTimeUpdate } from '../types/gtfs-rt'; export interface StopTimeUpdateFilters { @@ -11,7 +11,7 @@ export interface StopTimeUpdateFilters { /** * Parse stop time update from database row (includes trip_id and rt_last_updated) */ -function parseStopTimeUpdate(row: ParamsObject): StopTimeUpdate { +function parseStopTimeUpdate(row: Row): StopTimeUpdate { const stu: StopTimeUpdate = { stop_sequence: row.stop_sequence !== null ? Number(row.stop_sequence) : undefined, stop_id: row.stop_id ? String(row.stop_id) : undefined, @@ -46,11 +46,11 @@ function parseStopTimeUpdate(row: ParamsObject): StopTimeUpdate { * - Filters support both single values and arrays * - Returns stop time updates with trip_id and rt_last_updated populated */ -export function getStopTimeUpdates( - db: Database, +export async function getStopTimeUpdates( + db: GtfsDatabase, filters: StopTimeUpdateFilters = {}, stalenessThreshold: number = 120 -): StopTimeUpdate[] { +): Promise { const { tripId, stopId, stopSequence, limit } = filters; const conditions: string[] = []; @@ -105,18 +105,18 @@ export function getStopTimeUpdates( } // Execute query - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const stopTimeUpdates: StopTimeUpdate[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); stopTimeUpdates.push(parseStopTimeUpdate(row)); } - stmt.free(); + await stmt.free(); return stopTimeUpdates; } @@ -124,6 +124,6 @@ export function getStopTimeUpdates( * Get all stop time updates without staleness filtering (for debugging) * Convenience wrapper for getStopTimeUpdates() with no staleness threshold */ -export function getAllStopTimeUpdates(db: Database): StopTimeUpdate[] { +export async function getAllStopTimeUpdates(db: GtfsDatabase): Promise { return getStopTimeUpdates(db, {}, Number.MAX_SAFE_INTEGER); } diff --git a/src/queries/rt-trip-updates.ts b/src/queries/rt-trip-updates.ts index 7f0e63d..297e37c 100644 --- a/src/queries/rt-trip-updates.ts +++ b/src/queries/rt-trip-updates.ts @@ -1,4 +1,4 @@ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { TripUpdate, StopTimeUpdate } from '../types/gtfs-rt'; import { getStopTimeUpdates } from './rt-stop-time-updates'; @@ -12,7 +12,7 @@ export interface TripUpdateFilters { /** * Parse trip update from database row */ -export function parseTripUpdate(row: ParamsObject): TripUpdate { +export function parseTripUpdate(row: Row): TripUpdate { const tu: TripUpdate = { trip_id: String(row.trip_id), route_id: row.route_id ? String(row.route_id) : undefined, @@ -38,11 +38,11 @@ export function parseTripUpdate(row: ParamsObject): TripUpdate { /** * Get trip updates with optional filters */ -export function getTripUpdates( - db: Database, +export async function getTripUpdates( + db: GtfsDatabase, filters: TripUpdateFilters = {}, stalenessThreshold: number = 120 -): TripUpdate[] { +): Promise { const { tripId, routeId, vehicleId, limit } = filters; const conditions: string[] = []; @@ -85,25 +85,25 @@ export function getTripUpdates( } // Execute query - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const tripUpdates: TripUpdate[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); tripUpdates.push(parseTripUpdate(row)); } - stmt.free(); + await stmt.free(); // Populate stop_time_update arrays if (tripUpdates.length > 0) { const tripIds = tripUpdates.map(tu => tu.trip_id); // Query stop time updates for all retrieved trip IDs - const stopTimeUpdates = getStopTimeUpdates(db, { tripId: tripIds }, stalenessThreshold); + const stopTimeUpdates = await getStopTimeUpdates(db, { tripId: tripIds }, stalenessThreshold); // Group stop time updates by trip_id const stopTimesByTripId = new Map(); @@ -127,36 +127,36 @@ export function getTripUpdates( /** * Get trip update by trip ID */ -export function getTripUpdateByTripId( - db: Database, +export async function getTripUpdateByTripId( + db: GtfsDatabase, tripId: string, stalenessThreshold: number = 120 -): TripUpdate | null { - const updates = getTripUpdates(db, { tripId, limit: 1 }, stalenessThreshold); +): Promise { + const updates = await getTripUpdates(db, { tripId, limit: 1 }, stalenessThreshold); return updates.length > 0 ? updates[0] : null; } /** * Get all trip updates without staleness filtering (for debugging) */ -export function getAllTripUpdates(db: Database): TripUpdate[] { +export async function getAllTripUpdates(db: GtfsDatabase): Promise { const sql = 'SELECT * FROM rt_trip_updates ORDER BY rt_last_updated DESC'; - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); const tripUpdates: TripUpdate[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); tripUpdates.push(parseTripUpdate(row)); } - stmt.free(); + await stmt.free(); // Populate stop_time_update arrays (no staleness filtering for debug function) if (tripUpdates.length > 0) { const tripIds = tripUpdates.map(tu => tu.trip_id); // Query stop time updates for all retrieved trip IDs (use very large threshold to disable staleness filtering) - const stopTimeUpdates = getStopTimeUpdates(db, { tripId: tripIds }, Number.MAX_SAFE_INTEGER); + const stopTimeUpdates = await getStopTimeUpdates(db, { tripId: tripIds }, Number.MAX_SAFE_INTEGER); // Group stop time updates by trip_id const stopTimesByTripId = new Map(); diff --git a/src/queries/rt-vehicle-positions.ts b/src/queries/rt-vehicle-positions.ts index 69a4445..8753fff 100644 --- a/src/queries/rt-vehicle-positions.ts +++ b/src/queries/rt-vehicle-positions.ts @@ -1,4 +1,4 @@ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { VehiclePosition } from '../types/gtfs-rt'; import type { VehiclePositionFilters } from '../types/gtfs-rt'; @@ -7,7 +7,7 @@ export type { VehiclePositionFilters }; /** * Parse vehicle position from database row */ -export function parseVehiclePosition(row: ParamsObject): VehiclePosition { +export function parseVehiclePosition(row: Row): VehiclePosition { const vp: VehiclePosition = { trip_id: String(row.trip_id), route_id: row.route_id ? String(row.route_id) : undefined, @@ -60,11 +60,11 @@ export function parseVehiclePosition(row: ParamsObject): VehiclePosition { /** * Get vehicle positions with optional filters */ -export function getVehiclePositions( - db: Database, +export async function getVehiclePositions( + db: GtfsDatabase, filters: VehiclePositionFilters = {}, stalenessThreshold: number = 120 -): VehiclePosition[] { +): Promise { const { tripId, routeId, vehicleId, limit } = filters; const conditions: string[] = []; @@ -107,46 +107,46 @@ export function getVehiclePositions( } // Execute query - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const positions: VehiclePosition[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); positions.push(parseVehiclePosition(row)); } - stmt.free(); + await stmt.free(); return positions; } /** * Get vehicle position by trip ID */ -export function getVehiclePositionByTripId( - db: Database, +export async function getVehiclePositionByTripId( + db: GtfsDatabase, tripId: string, stalenessThreshold: number = 120 -): VehiclePosition | null { - const positions = getVehiclePositions(db, { tripId, limit: 1 }, stalenessThreshold); +): Promise { + const positions = await getVehiclePositions(db, { tripId, limit: 1 }, stalenessThreshold); return positions.length > 0 ? positions[0] : null; } /** * Get all vehicle positions without staleness filtering (for debugging) */ -export function getAllVehiclePositions(db: Database): VehiclePosition[] { +export async function getAllVehiclePositions(db: GtfsDatabase): Promise { const sql = 'SELECT * FROM rt_vehicle_positions ORDER BY rt_last_updated DESC'; - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); const positions: VehiclePosition[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); positions.push(parseVehiclePosition(row)); } - stmt.free(); + await stmt.free(); return positions; } diff --git a/src/queries/shapes.ts b/src/queries/shapes.ts index 0802672..a6096c7 100644 --- a/src/queries/shapes.ts +++ b/src/queries/shapes.ts @@ -2,7 +2,7 @@ * Shape Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Shape, Route } from '../types/gtfs'; export interface ShapeFilters { @@ -41,10 +41,10 @@ export interface GeoJsonFeatureCollection { * Get shapes with optional filters * - Filters support both single values and arrays */ -export function getShapes( - db: Database, +export async function getShapes( + db: GtfsDatabase, filters: ShapeFilters = {} -): Shape[] { +): Promise { const { shapeId, routeId, tripId, limit } = filters; // Determine if we need to join with trips table @@ -108,18 +108,18 @@ export function getShapes( params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const shapes: Shape[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); shapes.push(rowToShape(row)); } - stmt.free(); + await stmt.free(); return shapes; } @@ -130,12 +130,12 @@ export function getShapes( * @param filters - Same filters as getShapes * @param precision - Number of decimal places for coordinates (default: 6) */ -export function getShapesToGeojson( - db: Database, +export async function getShapesToGeojson( + db: GtfsDatabase, filters: ShapeFilters = {}, precision: number = 6 -): GeoJsonFeatureCollection { - const shapes = getShapes(db, filters); +): Promise { + const shapes = await getShapes(db, filters); // Group shapes by shape_id const shapeGroups = new Map(); @@ -149,7 +149,7 @@ export function getShapesToGeojson( } // Get route information for each shape - const shapeRouteMap = getRoutesByShapeIds(db, Array.from(shapeGroups.keys())); + const shapeRouteMap = await getRoutesByShapeIds(db, Array.from(shapeGroups.keys())); // Build GeoJSON features const features: GeoJsonFeature[] = []; @@ -200,10 +200,10 @@ export function getShapesToGeojson( /** * Get the first matching route for each shape_id */ -function getRoutesByShapeIds( - db: Database, +async function getRoutesByShapeIds( + db: GtfsDatabase, shapeIds: string[] -): Map { +): Promise> { if (shapeIds.length === 0) { return new Map(); } @@ -219,17 +219,17 @@ function getRoutesByShapeIds( GROUP BY t.shape_id `; - const stmt = db.prepare(sql); - stmt.bind(shapeIds); + const stmt = await db.prepare(sql); + await stmt.bind(shapeIds); const result = new Map(); - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); const shapeId = String(row.shape_id); result.set(shapeId, rowToRoute(row)); } - stmt.free(); + await stmt.free(); return result; } @@ -237,7 +237,7 @@ function getRoutesByShapeIds( /** * Convert database row to Shape object */ -function rowToShape(row: ParamsObject): Shape { +function rowToShape(row: Row): Shape { return { shape_id: String(row.shape_id), shape_pt_lat: Number(row.shape_pt_lat), @@ -250,7 +250,7 @@ function rowToShape(row: ParamsObject): Shape { /** * Convert database row to Route object */ -function rowToRoute(row: ParamsObject): Route { +function rowToRoute(row: Row): Route { return { route_id: String(row.route_id), route_short_name: row.route_short_name ? String(row.route_short_name) : '', diff --git a/src/queries/stop-times.ts b/src/queries/stop-times.ts index d0e443f..30ba248 100644 --- a/src/queries/stop-times.ts +++ b/src/queries/stop-times.ts @@ -2,7 +2,7 @@ * Stop Time Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { StopTime, Stop } from '../types/gtfs'; import type { StopTimeRealtime } from '../types/gtfs-rt'; import type { PickupDropOffType } from '../types/gtfs-enums'; @@ -38,11 +38,11 @@ export interface StopTimeWithRealtime extends StopTime { /** * Merge realtime data with stop times */ -function mergeRealtimeData( +async function mergeRealtimeData( stopTimes: StopTime[], - db: Database, + db: GtfsDatabase, stalenessThreshold: number -): StopTimeWithRealtime[] { +): Promise { const now = Math.floor(Date.now() / 1000); const staleThreshold = now - stalenessThreshold; @@ -51,19 +51,19 @@ function mergeRealtimeData( if (tripIds.length === 0) return stopTimes; const placeholders = tripIds.map(() => '?').join(', '); - const stmt = db.prepare(` + const stmt = await db.prepare(` SELECT trip_id, stop_sequence, stop_id, arrival_delay, arrival_time, departure_delay, departure_time, schedule_relationship FROM rt_stop_time_updates WHERE trip_id IN (${placeholders}) AND rt_last_updated >= ? `); - stmt.bind([...tripIds, staleThreshold]); + await stmt.bind([...tripIds, staleThreshold]); // Build map of trip_id+stop_sequence -> RT data const rtMap = new Map(); - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); const key = `${row.trip_id}_${row.stop_sequence}`; rtMap.set(key, { arrival_delay: row.arrival_delay !== null ? Number(row.arrival_delay) : undefined, @@ -73,7 +73,7 @@ function mergeRealtimeData( schedule_relationship: row.schedule_relationship !== null ? Number(row.schedule_relationship) : undefined }); } - stmt.free(); + await stmt.free(); // Merge RT data with stop times return stopTimes.map((st): StopTimeWithRealtime => { @@ -91,11 +91,11 @@ function mergeRealtimeData( * Get stop times with optional filters * - Filters support both single values and arrays */ -export function getStopTimes( - db: Database, +export async function getStopTimes( + db: GtfsDatabase, filters: StopTimeFilters = {}, stalenessThreshold: number = 120 -): StopTime[] | StopTimeWithRealtime[] { +): Promise { const { tripId, stopId, routeId, serviceIds, directionId, agencyId, pickupType, dropOffType, includeRealtime, limit } = filters; // Determine if we need to join with trips and/or routes table @@ -202,18 +202,18 @@ export function getStopTimes( params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const stopTimes: StopTime[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); stopTimes.push(rowToStopTime(row)); } - stmt.free(); + await stmt.free(); // Merge realtime data if requested if (includeRealtime) { @@ -243,26 +243,26 @@ export function getStopTimes( * @param tripIds - Array of trip IDs to analyze * @returns Ordered array of Stop objects representing all unique stops */ -export function buildOrderedStopList(db: Database, tripIds: string[]): Stop[] { +export async function buildOrderedStopList(db: GtfsDatabase, tripIds: string[]): Promise { if (tripIds.length === 0) { return []; } // Fetch all stop times for the given trips, ordered by trip and sequence const placeholders = tripIds.map(() => '?').join(', '); - const stmt = db.prepare(` + const stmt = await db.prepare(` SELECT trip_id, stop_id, stop_sequence FROM stop_times WHERE trip_id IN (${placeholders}) ORDER BY trip_id, stop_sequence `); - stmt.bind(tripIds); + await stmt.bind(tripIds); // Group stop times by trip const tripStopSequences = new Map>(); - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); const tripId = String(row.trip_id); const stopId = String(row.stop_id); const stopSequence = Number(row.stop_sequence); @@ -272,7 +272,7 @@ export function buildOrderedStopList(db: Database, tripIds: string[]): Stop[] { } tripStopSequences.get(tripId)?.push({ stop_id: stopId, stop_sequence: stopSequence }); } - stmt.free(); + await stmt.free(); // Build the ordered list of stop IDs const orderedStopIds: string[] = []; @@ -308,7 +308,7 @@ export function buildOrderedStopList(db: Database, tripIds: string[]): Stop[] { } // Get all stops in a single query - const stops = getStops(db, { stopId: orderedStopIds }); + const stops = await getStops(db, { stopId: orderedStopIds }); // Create a map for quick lookup const stopMap = new Map(); @@ -377,7 +377,7 @@ function findInsertionPosition( /** * Convert database row to StopTime object */ -function rowToStopTime(row: ParamsObject): StopTime { +function rowToStopTime(row: Row): StopTime { return { trip_id: String(row.trip_id), arrival_time: row.arrival_time ? String(row.arrival_time) : undefined, diff --git a/src/queries/stops.ts b/src/queries/stops.ts index b32723d..5907eaf 100644 --- a/src/queries/stops.ts +++ b/src/queries/stops.ts @@ -2,7 +2,7 @@ * Stop Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Stop } from '../types/gtfs'; export interface StopFilters { @@ -19,7 +19,7 @@ export interface StopFilters { * - Use name filter for partial name matching * - Use tripId filter to get stops for a specific trip (ordered by stop_sequence) */ -export function getStops(db: Database, filters: StopFilters = {}): Stop[] { +export async function getStops(db: GtfsDatabase, filters: StopFilters = {}): Promise { const { stopId, stopCode, name, tripId, limit } = filters; // Handle special case: get stops by trip (requires JOIN) @@ -28,21 +28,21 @@ export function getStops(db: Database, filters: StopFilters = {}): Stop[] { if (tripIds.length === 0) return []; const placeholders = tripIds.map(() => '?').join(', '); - const stmt = db.prepare(` + const stmt = await db.prepare(` SELECT s.* FROM stops s INNER JOIN stop_times st ON s.stop_id = st.stop_id WHERE st.trip_id IN (${placeholders}) ORDER BY st.stop_sequence `); - stmt.bind(tripIds); + await stmt.bind(tripIds); const stops: Stop[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); stops.push(rowToStop(row)); } - stmt.free(); + await stmt.free(); return stops; } @@ -84,18 +84,18 @@ export function getStops(db: Database, filters: StopFilters = {}): Stop[] { params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const stops: Stop[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); stops.push(rowToStop(row)); } - stmt.free(); + await stmt.free(); return stops; } @@ -103,14 +103,14 @@ export function getStops(db: Database, filters: StopFilters = {}): Stop[] { * Search stops by name (case-insensitive, partial match) * This is a convenience method for name-based searches */ -export function searchStopsByName(db: Database, name: string, limit = 50): Stop[] { +export async function searchStopsByName(db: GtfsDatabase, name: string, limit = 50): Promise { return getStops(db, { name, limit }); } /** * Convert database row to Stop object */ -function rowToStop(row: ParamsObject): Stop { +function rowToStop(row: Row): Stop { return { stop_id: String(row.stop_id), stop_name: String(row.stop_name), diff --git a/src/queries/trips.ts b/src/queries/trips.ts index cd89a03..f0f02e3 100644 --- a/src/queries/trips.ts +++ b/src/queries/trips.ts @@ -2,7 +2,7 @@ * Trip Query Methods */ -import type { Database, ParamsObject } from 'sql.js'; +import type { GtfsDatabase, Row } from '../adapters/types'; import type { Trip } from '../types/gtfs'; import type { TripRealtime, VehiclePosition } from '../types/gtfs-rt'; import { parseVehiclePosition } from './rt-vehicle-positions'; @@ -24,11 +24,11 @@ export interface TripWithRealtime extends Trip { /** * Merge realtime data with trips */ -function mergeRealtimeData( +async function mergeRealtimeData( trips: Trip[], - db: Database, + db: GtfsDatabase, stalenessThreshold: number -): TripWithRealtime[] { +): Promise { const now = Math.floor(Date.now() / 1000); const staleThreshold = now - stalenessThreshold; @@ -38,39 +38,39 @@ function mergeRealtimeData( const placeholders = tripIds.map(() => '?').join(', '); // Get vehicle positions - const vpStmt = db.prepare(` + const vpStmt = await db.prepare(` SELECT * FROM rt_vehicle_positions WHERE trip_id IN (${placeholders}) AND rt_last_updated >= ? `); - vpStmt.bind([...tripIds, staleThreshold]); + await vpStmt.bind([...tripIds, staleThreshold]); const vpMap = new Map(); - while (vpStmt.step()) { - const row = vpStmt.getAsObject(); + while (await vpStmt.step()) { + const row = await vpStmt.getAsObject(); const vp = parseVehiclePosition(row); vpMap.set(vp.trip_id, vp); } - vpStmt.free(); + await vpStmt.free(); // Get trip updates - const tuStmt = db.prepare(` + const tuStmt = await db.prepare(` SELECT * FROM rt_trip_updates WHERE trip_id IN (${placeholders}) AND rt_last_updated >= ? `); - tuStmt.bind([...tripIds, staleThreshold]); + await tuStmt.bind([...tripIds, staleThreshold]); const tuMap = new Map(); - while (tuStmt.step()) { - const row = tuStmt.getAsObject(); + while (await tuStmt.step()) { + const row = await tuStmt.getAsObject(); const tripId = String(row.trip_id); tuMap.set(tripId, { delay: row.delay !== null ? Number(row.delay) : undefined, schedule_relationship: row.schedule_relationship !== null ? Number(row.schedule_relationship) : undefined }); } - tuStmt.free(); + await tuStmt.free(); // Merge realtime data return trips.map((trip): TripWithRealtime => { @@ -95,11 +95,11 @@ function mergeRealtimeData( * Get trips with optional filters * - Filters support both single values and arrays */ -export function getTrips( - db: Database, +export async function getTrips( + db: GtfsDatabase, filters: TripFilters = {}, stalenessThreshold: number = 120 -): Trip[] | TripWithRealtime[] { +): Promise { const { tripId, routeId, serviceIds, directionId, agencyId, includeRealtime, limit } = filters; // Determine if we need to join with routes table @@ -166,18 +166,18 @@ export function getTrips( params.push(limit); } - const stmt = db.prepare(sql); + const stmt = await db.prepare(sql); if (params.length > 0) { - stmt.bind(params); + await stmt.bind(params); } const trips: Trip[] = []; - while (stmt.step()) { - const row = stmt.getAsObject(); + while (await stmt.step()) { + const row = await stmt.getAsObject(); trips.push(rowToTrip(row)); } - stmt.free(); + await stmt.free(); // Merge realtime data if requested if (includeRealtime) { @@ -190,7 +190,7 @@ export function getTrips( /** * Convert database row to Trip object */ -function rowToTrip(row: ParamsObject): Trip { +function rowToTrip(row: Row): Trip { return { trip_id: String(row.trip_id), route_id: String(row.route_id), diff --git a/src/schema/gtfs-rt-schema.ts b/src/schema/gtfs-rt-schema.ts index 0251e0b..75a2f4b 100644 --- a/src/schema/gtfs-rt-schema.ts +++ b/src/schema/gtfs-rt-schema.ts @@ -1,11 +1,11 @@ -import type { Database } from 'sql.js'; +import type { GtfsDatabase } from '../adapters/types'; /** * Create GTFS Realtime tables in the database */ -export function createRealtimeTables(db: Database): void { +export async function createRealtimeTables(db: GtfsDatabase): Promise { // Alerts table - db.run(` + await db.run(` CREATE TABLE IF NOT EXISTS rt_alerts ( id TEXT PRIMARY KEY, active_period TEXT, -- JSON array of TimeRange objects @@ -20,10 +20,10 @@ export function createRealtimeTables(db: Database): void { `); // Create index on rt_last_updated for staleness filtering - db.run('CREATE INDEX IF NOT EXISTS idx_rt_alerts_updated ON rt_alerts(rt_last_updated)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_alerts_updated ON rt_alerts(rt_last_updated)'); // Vehicle Positions table - db.run(` + await db.run(` CREATE TABLE IF NOT EXISTS rt_vehicle_positions ( trip_id TEXT PRIMARY KEY, route_id TEXT, @@ -45,11 +45,11 @@ export function createRealtimeTables(db: Database): void { ) `); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_vehicle_positions_updated ON rt_vehicle_positions(rt_last_updated)'); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_vehicle_positions_route ON rt_vehicle_positions(route_id)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_vehicle_positions_updated ON rt_vehicle_positions(rt_last_updated)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_vehicle_positions_route ON rt_vehicle_positions(route_id)'); // Trip Updates table - db.run(` + await db.run(` CREATE TABLE IF NOT EXISTS rt_trip_updates ( trip_id TEXT PRIMARY KEY, route_id TEXT, @@ -63,11 +63,11 @@ export function createRealtimeTables(db: Database): void { ) `); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_trip_updates_updated ON rt_trip_updates(rt_last_updated)'); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_trip_updates_route ON rt_trip_updates(route_id)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_trip_updates_updated ON rt_trip_updates(rt_last_updated)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_trip_updates_route ON rt_trip_updates(route_id)'); // Stop Time Updates table (child of trip updates) - db.run(` + await db.run(` CREATE TABLE IF NOT EXISTS rt_stop_time_updates ( trip_id TEXT NOT NULL, stop_sequence INTEGER, @@ -85,16 +85,16 @@ export function createRealtimeTables(db: Database): void { ) `); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_stop_time_updates_updated ON rt_stop_time_updates(rt_last_updated)'); - db.run('CREATE INDEX IF NOT EXISTS idx_rt_stop_time_updates_stop ON rt_stop_time_updates(stop_id)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_stop_time_updates_updated ON rt_stop_time_updates(rt_last_updated)'); + await db.run('CREATE INDEX IF NOT EXISTS idx_rt_stop_time_updates_stop ON rt_stop_time_updates(stop_id)'); } /** * Clear all realtime data from the database */ -export function clearRealtimeData(db: Database): void { - db.run('DELETE FROM rt_alerts'); - db.run('DELETE FROM rt_vehicle_positions'); - db.run('DELETE FROM rt_trip_updates'); - db.run('DELETE FROM rt_stop_time_updates'); +export async function clearRealtimeData(db: GtfsDatabase): Promise { + await db.run('DELETE FROM rt_alerts'); + await db.run('DELETE FROM rt_vehicle_positions'); + await db.run('DELETE FROM rt_trip_updates'); + await db.run('DELETE FROM rt_stop_time_updates'); } diff --git a/tests/build-ordered-stop-list.test.ts b/tests/build-ordered-stop-list.test.ts index 56287dd..1bafa0f 100644 --- a/tests/build-ordered-stop-list.test.ts +++ b/tests/build-ordered-stop-list.test.ts @@ -1,12 +1,10 @@ /** * Tests for buildOrderedStopList method - * - * This method builds an optimal ordered list of stops from multiple trips - * that may have different stop patterns (express vs local, different start/end points) */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { GtfsSqlJs } from '../src/gtfs-sqljs'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; import path from 'path'; import fs from 'fs/promises'; import initSqlJs from 'sql.js'; @@ -18,26 +16,28 @@ describe('buildOrderedStopList', () => { // Load the sample GTFS feed const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); const zipData = await fs.readFile(feedPath); - gtfs = await GtfsSqlJs.fromZipData(zipData); + gtfs = await GtfsSqlJs.fromZipData(zipData, { + adapter: await createSqlJsAdapter(), + }); }); - afterAll(() => { - gtfs?.close(); + afterAll(async () => { + await gtfs?.close(); }); describe('Edge Cases', () => { - it('should return empty array for empty trip list', () => { - const stops = gtfs.buildOrderedStopList([]); + it('should return empty array for empty trip list', async () => { + const stops = await gtfs.buildOrderedStopList([]); expect(stops).toEqual([]); }); - it('should return empty array for non-existent trip IDs', () => { - const stops = gtfs.buildOrderedStopList(['NONEXISTENT_TRIP']); + it('should return empty array for non-existent trip IDs', async () => { + const stops = await gtfs.buildOrderedStopList(['NONEXISTENT_TRIP']); expect(stops).toEqual([]); }); - it('should handle single trip', () => { - const stops = gtfs.buildOrderedStopList(['AB1']); + it('should handle single trip', async () => { + const stops = await gtfs.buildOrderedStopList(['AB1']); expect(stops.length).toBe(2); expect(stops[0].stop_id).toBe('BEATTY_AIRPORT'); @@ -46,13 +46,13 @@ describe('buildOrderedStopList', () => { }); describe('Simple Cases - Same Route Same Direction', () => { - it('should handle two trips with identical stop sequences', () => { + it('should handle two trips with identical stop sequences', async () => { // Get trips for route CITY going in direction 0 - const trips = gtfs.getTrips({ routeId: 'CITY', directionId: 0 }); + const trips = await gtfs.getTrips({ routeId: 'CITY', directionId: 0 }); const tripIds = trips.map(t => t.trip_id); // Build ordered stop list - const stops = gtfs.buildOrderedStopList(tripIds); + const stops = await gtfs.buildOrderedStopList(tripIds); // Verify we get all unique stops expect(stops.length).toBeGreaterThan(0); @@ -63,9 +63,8 @@ describe('buildOrderedStopList', () => { expect(stopIds.length).toBe(uniqueStopIds.length); // Verify stops are in a valid order - // Each individual trip should have its stops in the same order as the result for (const tripId of tripIds) { - const tripStops = gtfs.getStops({ tripId }); + const tripStops = await gtfs.getStops({ tripId }); const tripStopIds = tripStops.map(s => s.stop_id); // Extract the positions of this trip's stops in the result @@ -80,14 +79,12 @@ describe('buildOrderedStopList', () => { }); describe('Real-World Scenarios', () => { - it('should handle route AB (both directions)', () => { - // Route AB has trips going both directions - const trips = gtfs.getTrips({ routeId: 'AB' }); + it('should handle route AB (both directions)', async () => { + const trips = await gtfs.getTrips({ routeId: 'AB' }); const tripIds = trips.map(t => t.trip_id); - const stops = gtfs.buildOrderedStopList(tripIds); + const stops = await gtfs.buildOrderedStopList(tripIds); - // Should have both stops expect(stops.length).toBe(2); const stopIds = stops.map(s => s.stop_id); @@ -95,11 +92,11 @@ describe('buildOrderedStopList', () => { expect(stopIds).toContain('BULLFROG'); }); - it('should handle route CITY trips', () => { - const trips = gtfs.getTrips({ routeId: 'CITY' }); + it('should handle route CITY trips', async () => { + const trips = await gtfs.getTrips({ routeId: 'CITY' }); const tripIds = trips.map(t => t.trip_id); - const stops = gtfs.buildOrderedStopList(tripIds); + const stops = await gtfs.buildOrderedStopList(tripIds); // Verify we have multiple stops expect(stops.length).toBeGreaterThan(0); @@ -113,14 +110,14 @@ describe('buildOrderedStopList', () => { }); }); - it('should maintain correct ordering for CITY route direction 0', () => { - const trips = gtfs.getTrips({ routeId: 'CITY', directionId: 0 }); + it('should maintain correct ordering for CITY route direction 0', async () => { + const trips = await gtfs.getTrips({ routeId: 'CITY', directionId: 0 }); const tripIds = trips.map(t => t.trip_id); - const stops = gtfs.buildOrderedStopList(tripIds); + const stops = await gtfs.buildOrderedStopList(tripIds); // Get stops for first trip to compare - const firstTripStops = gtfs.getStops({ tripId: tripIds[0] }); + const firstTripStops = await gtfs.getStops({ tripId: tripIds[0] }); const firstTripStopIds = firstTripStops.map(s => s.stop_id); // All stops from first trip should appear in result in the same order @@ -137,7 +134,6 @@ describe('buildOrderedStopList', () => { let customGtfs: GtfsSqlJs; beforeAll(async () => { - // Create a custom GTFS database with specific test scenarios const SQL = await initSqlJs(); const db = new SQL.Database(); @@ -215,16 +211,17 @@ describe('buildOrderedStopList', () => { ); }); - // Initialize GtfsSqlJs with the populated database - customGtfs = await GtfsSqlJs.fromDatabase(db.export()); + customGtfs = await GtfsSqlJs.fromDatabase(db.export().buffer, { + adapter: await createSqlJsAdapter({ SQL }), + }); }); - afterAll(() => { - customGtfs?.close(); + afterAll(async () => { + await customGtfs?.close(); }); - it('should merge local and express trips correctly', () => { - const stops = customGtfs.buildOrderedStopList(['LOCAL', 'EXPRESS']); + it('should merge local and express trips correctly', async () => { + const stops = await customGtfs.buildOrderedStopList(['LOCAL', 'EXPRESS']); const stopIds = stops.map(s => s.stop_id); @@ -232,21 +229,19 @@ describe('buildOrderedStopList', () => { expect(stopIds).toEqual(['A', 'B', 'C', 'D', 'E', 'F']); }); - it('should handle trips with different start/end points', () => { - const stops = customGtfs.buildOrderedStopList(['LOCAL', 'SHORT']); + it('should handle trips with different start/end points', async () => { + const stops = await customGtfs.buildOrderedStopList(['LOCAL', 'SHORT']); const stopIds = stops.map(s => s.stop_id); - // Should have all stops from both trips in correct order expect(stopIds).toEqual(['A', 'B', 'C', 'D', 'E', 'F']); }); - it('should correctly insert intermediate stops', () => { - const stops = customGtfs.buildOrderedStopList(['LOCAL', 'WITH_X']); + it('should correctly insert intermediate stops', async () => { + const stops = await customGtfs.buildOrderedStopList(['LOCAL', 'WITH_X']); const stopIds = stops.map(s => s.stop_id); - // X should appear between B and C const xIndex = stopIds.indexOf('X'); const bIndex = stopIds.indexOf('B'); const cIndex = stopIds.indexOf('C'); @@ -254,37 +249,33 @@ describe('buildOrderedStopList', () => { expect(xIndex).toBeGreaterThan(bIndex); expect(xIndex).toBeLessThan(cIndex); - // Should maintain overall order: A-B-X-C-D-E-F expect(stopIds).toEqual(['A', 'B', 'X', 'C', 'D', 'E', 'F']); }); - it('should handle all four trips together', () => { - const stops = customGtfs.buildOrderedStopList(['LOCAL', 'EXPRESS', 'SHORT', 'WITH_X']); + it('should handle all four trips together', async () => { + const stops = await customGtfs.buildOrderedStopList(['LOCAL', 'EXPRESS', 'SHORT', 'WITH_X']); const stopIds = stops.map(s => s.stop_id); - // Should have all 7 unique stops in correct order expect(stopIds).toEqual(['A', 'B', 'X', 'C', 'D', 'E', 'F']); }); - it('should maintain order regardless of trip order', () => { - // Test with different orderings of the same trips - const stops1 = customGtfs.buildOrderedStopList(['EXPRESS', 'LOCAL', 'SHORT', 'WITH_X']); - const stops2 = customGtfs.buildOrderedStopList(['SHORT', 'WITH_X', 'EXPRESS', 'LOCAL']); - const stops3 = customGtfs.buildOrderedStopList(['WITH_X', 'SHORT', 'LOCAL', 'EXPRESS']); + it('should maintain order regardless of trip order', async () => { + const stops1 = await customGtfs.buildOrderedStopList(['EXPRESS', 'LOCAL', 'SHORT', 'WITH_X']); + const stops2 = await customGtfs.buildOrderedStopList(['SHORT', 'WITH_X', 'EXPRESS', 'LOCAL']); + const stops3 = await customGtfs.buildOrderedStopList(['WITH_X', 'SHORT', 'LOCAL', 'EXPRESS']); const stopIds1 = stops1.map(s => s.stop_id); const stopIds2 = stops2.map(s => s.stop_id); const stopIds3 = stops3.map(s => s.stop_id); - // All should produce the same ordering expect(stopIds1).toEqual(['A', 'B', 'X', 'C', 'D', 'E', 'F']); expect(stopIds2).toEqual(['A', 'B', 'X', 'C', 'D', 'E', 'F']); expect(stopIds3).toEqual(['A', 'B', 'X', 'C', 'D', 'E', 'F']); }); - it('should return full Stop objects with all properties', () => { - const stops = customGtfs.buildOrderedStopList(['LOCAL']); + it('should return full Stop objects with all properties', async () => { + const stops = await customGtfs.buildOrderedStopList(['LOCAL']); expect(stops.length).toBe(6); @@ -302,7 +293,6 @@ describe('buildOrderedStopList', () => { let complexGtfs: GtfsSqlJs; beforeAll(async () => { - // Create a more complex scenario mimicking real transit patterns const SQL = await initSqlJs(); const db = new SQL.Database(); @@ -371,35 +361,28 @@ describe('buildOrderedStopList', () => { ); } - // Initialize GtfsSqlJs with the populated database - complexGtfs = await GtfsSqlJs.fromDatabase(db.export()); + complexGtfs = await GtfsSqlJs.fromDatabase(db.export().buffer, { + adapter: await createSqlJsAdapter({ SQL }), + }); }); - afterAll(() => { - complexGtfs?.close(); + afterAll(async () => { + await complexGtfs?.close(); }); - it('should correctly merge trips with different patterns', () => { - const stops = complexGtfs.buildOrderedStopList(['T1', 'T2', 'T3']); + it('should correctly merge trips with different patterns', async () => { + const stops = await complexGtfs.buildOrderedStopList(['T1', 'T2', 'T3']); const stopIds = stops.map(s => s.stop_id); - // Should have all 10 stops in order expect(stopIds).toEqual(['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7', 'S8', 'S9', 'S10']); }); - it('should handle subset of trips', () => { - const stops = complexGtfs.buildOrderedStopList(['T2', 'T3']); + it('should handle subset of trips', async () => { + const stops = await complexGtfs.buildOrderedStopList(['T2', 'T3']); const stopIds = stops.map(s => s.stop_id); - // T2 has: S2, S4, S6, S8, S10 - // T3 has: S3, S4, S5, S6, S7 - // Both trips share S4, S6 - // Without a trip containing both S2 and S3, their relative order depends on processing order - // The result should maintain the ordering constraints from each individual trip - - // Verify all expected stops are present expect(stopIds).toContain('S2'); expect(stopIds).toContain('S3'); expect(stopIds).toContain('S4'); diff --git a/tests/e2e-better-sqlite3.test.ts b/tests/e2e-better-sqlite3.test.ts new file mode 100644 index 0000000..efe3fa9 --- /dev/null +++ b/tests/e2e-better-sqlite3.test.ts @@ -0,0 +1,127 @@ +/** + * End-to-end test exercising the built-in better-sqlite3 adapter. + * + * Loads the sample GTFS feed into a better-sqlite3 database via + * `GtfsSqlJs.attach()`, runs the same query assertions as the sql.js tests, + * and verifies the cache layer gracefully no-ops when the adapter's + * `export()` throws `ExportNotSupportedError`. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import BetterSqlite3 from 'better-sqlite3'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs/promises'; +import { GtfsSqlJs } from '../src/gtfs-sqljs'; +import { ExportNotSupportedError } from '../src/adapters/types'; +import { loadGTFSZip } from '../src/loaders/zip-loader'; +import { loadGTFSData } from '../src/loaders/data-loader'; +import { wrapBetterSqlite3 } from '../src/adapters/better-sqlite3'; +import type { CacheMetadata, CacheStore } from '../src/cache/types'; + +describe('better-sqlite3 adapter — end-to-end', () => { + let tmpDir: string; + let dbPath: string; + let rawDb: BetterSqlite3.Database; + let gtfs: GtfsSqlJs; + + beforeAll(async () => { + tmpDir = mkdtempSync(path.join(tmpdir(), 'gtfs-bsqlite3-')); + dbPath = path.join(tmpDir, 'gtfs.db'); + rawDb = new BetterSqlite3(dbPath); + + const db = wrapBetterSqlite3(rawDb); + gtfs = await GtfsSqlJs.attach(db); + + // Load sample fixture via the underlying adapter surface — this is the + // most direct test of the adapter interface. + const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); + const zipData = await fs.readFile(feedPath); + const files = await loadGTFSZip(zipData); + await loadGTFSData(db, files); + }); + + afterAll(async () => { + await gtfs?.close(); + // `attach()` without ownsDatabase means we still own rawDb. + if (rawDb.open) rawDb.close(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('loads the sample feed and answers the same queries as sql.js', async () => { + const agencies = await gtfs.getAgencies({ agencyId: 'DTA' }); + expect(agencies.length).toBe(1); + expect(agencies[0].agency_name).toBe('Demo Transit Authority'); + + const stops = await gtfs.getStops(); + expect(stops.length).toBe(9); + + const routes = await gtfs.getRoutes(); + expect(routes.length).toBe(5); + + const ab1 = (await gtfs.getTrips({ tripId: 'AB1' }))[0]; + expect(ab1.route_id).toBe('AB'); + expect(ab1.trip_headsign).toBe('to Bullfrog'); + + const stopTimes = await gtfs.getStopTimes({ tripId: 'AB1' }); + expect(stopTimes.length).toBe(2); + expect(stopTimes[0].stop_id).toBe('BEATTY_AIRPORT'); + expect(stopTimes[1].stop_id).toBe('BULLFROG'); + }); + + it('exports succeed on an in-memory DB (better-sqlite3.serialize)', async () => { + // This same adapter wrapped around an in-memory better-sqlite3 DB must + // serialize cleanly. We don't call gtfs.export() here because the + // outer fixture DB is file-backed; we exercise the codepath directly. + const mem = new BetterSqlite3(':memory:'); + mem.exec('CREATE TABLE x (id INTEGER); INSERT INTO x VALUES (1);'); + const wrapped = wrapBetterSqlite3(mem); + const bytes = await wrapped.export(); + expect(bytes).toBeInstanceOf(Uint8Array); + expect(bytes.byteLength).toBeGreaterThan(0); + mem.close(); + }); + + it('cache layer catches ExportNotSupportedError and does not throw', async () => { + // A minimal in-memory cache store; we only care that `set()` is not called. + let setCalls = 0; + const cache: CacheStore = { + get: async () => null, + set: async (_key: string, _data: ArrayBuffer, _metadata: CacheMetadata) => { + setCalls++; + }, + delete: async () => { /* noop */ }, + clear: async () => { /* noop */ }, + }; + + // Build a fresh wrapped DB that throws from export() on purpose. + const throwingDb = wrapBetterSqlite3(new BetterSqlite3(':memory:')); + throwingDb.export = async () => { + throw new ExportNotSupportedError('simulated file-backed driver'); + }; + + // Factory adapter that returns the throwing DB for createEmpty(). + const throwingAdapter = { + createEmpty: async () => throwingDb, + openFromBuffer: async () => throwingDb, + }; + + const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); + const zipData = await fs.readFile(feedPath); + + const instance = await GtfsSqlJs.fromZipData(zipData, { + adapter: throwingAdapter, + cache, + }); + + // Cache write should have been skipped (no throw, no call to set). + expect(setCalls).toBe(0); + await instance.close(); + }); + + it('file remains on disk after close when caller owns the handle', async () => { + const stat = await fs.stat(dbPath); + expect(stat.size).toBeGreaterThan(0); + }); +}); diff --git a/tests/gtfs-sqljs.test.ts b/tests/gtfs-sqljs.test.ts index ee712bf..15fd151 100644 --- a/tests/gtfs-sqljs.test.ts +++ b/tests/gtfs-sqljs.test.ts @@ -3,13 +3,14 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import initSqlJs from 'sql.js'; +import initSqlJs, { type SqlJsStatic } from 'sql.js'; import { GtfsSqlJs } from '../src/gtfs-sqljs'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; import { createTestDatabase } from './helpers/test-database'; describe('GtfsSqlJs', () => { let gtfs: GtfsSqlJs; - let SQL: any; + let SQL: SqlJsStatic; beforeAll(async () => { // Initialize SQL.js @@ -19,184 +20,189 @@ describe('GtfsSqlJs', () => { const dbBuffer = await createTestDatabase(SQL); // Initialize GtfsSqlJs with test data - gtfs = await GtfsSqlJs.fromDatabase(dbBuffer, { SQL }); + gtfs = await GtfsSqlJs.fromDatabase(dbBuffer, { + adapter: await createSqlJsAdapter({ SQL }), + }); }); - afterAll(() => { - gtfs?.close(); + afterAll(async () => { + await gtfs?.close(); }); describe('Stop methods', () => { - it('should get stop by ID using filters', () => { - const stops = gtfs.getStops({ stopId: 'STOP1' }); + it('should get stop by ID using filters', async () => { + const stops = await gtfs.getStops({ stopId: 'STOP1' }); expect(stops.length).toBe(1); expect(stops[0].stop_id).toBe('STOP1'); expect(stops[0].stop_name).toBe('First Street'); }); - it('should get stop by code', () => { - const stops = gtfs.getStops({ stopCode: 'FS' }); + it('should get stop by code', async () => { + const stops = await gtfs.getStops({ stopCode: 'FS' }); expect(stops.length).toBe(1); expect(stops[0].stop_id).toBe('STOP1'); }); - it('should search stops by name', () => { - const stops = gtfs.getStops({ name: 'Street' }); + it('should search stops by name', async () => { + const stops = await gtfs.getStops({ name: 'Street' }); expect(stops.length).toBeGreaterThan(0); expect(stops[0].stop_name).toContain('Street'); }); - it('should return empty array for non-existent stop ID', () => { - const stops = gtfs.getStops({ stopId: 'NONEXISTENT' }); + it('should return empty array for non-existent stop ID', async () => { + const stops = await gtfs.getStops({ stopId: 'NONEXISTENT' }); expect(stops.length).toBe(0); }); - it('should get all stops', () => { - const stops = gtfs.getStops(); + it('should get all stops', async () => { + const stops = await gtfs.getStops(); expect(stops.length).toBeGreaterThan(0); }); - it('should get multiple stops by ID array', () => { - const stops = gtfs.getStops({ stopId: ['STOP1', 'STOP2'] }); + it('should get multiple stops by ID array', async () => { + const stops = await gtfs.getStops({ stopId: ['STOP1', 'STOP2'] }); expect(stops.length).toBe(2); }); }); describe('Route methods', () => { - it('should get route by ID using filters', () => { - const routes = gtfs.getRoutes({ routeId: 'ROUTE1' }); + it('should get route by ID using filters', async () => { + const routes = await gtfs.getRoutes({ routeId: 'ROUTE1' }); expect(routes.length).toBe(1); expect(routes[0].route_id).toBe('ROUTE1'); expect(routes[0].route_short_name).toBe('1'); }); - it('should get all routes', () => { - const routes = gtfs.getRoutes(); + it('should get all routes', async () => { + const routes = await gtfs.getRoutes(); expect(routes.length).toBeGreaterThan(0); }); - it('should return empty array for non-existent route', () => { - const routes = gtfs.getRoutes({ routeId: 'NONEXISTENT' }); + it('should return empty array for non-existent route', async () => { + const routes = await gtfs.getRoutes({ routeId: 'NONEXISTENT' }); expect(routes.length).toBe(0); }); - it('should get multiple routes by ID array', () => { - const routes = gtfs.getRoutes({ routeId: ['ROUTE1', 'ROUTE2'] }); + it('should get multiple routes by ID array', async () => { + const routes = await gtfs.getRoutes({ routeId: ['ROUTE1', 'ROUTE2'] }); expect(routes.length).toBeGreaterThanOrEqual(1); }); }); describe('Calendar methods', () => { - it('should get active service IDs for a date', () => { - const serviceIds = gtfs.getActiveServiceIds('20240101'); + it('should get active service IDs for a date', async () => { + const serviceIds = await gtfs.getActiveServiceIds('20240101'); expect(serviceIds.length).toBeGreaterThan(0); expect(serviceIds).toContain('WEEKDAY'); }); - it('should get calendar by service ID', () => { - const calendar = gtfs.getCalendarByServiceId('WEEKDAY'); + it('should get calendar by service ID', async () => { + const calendar = await gtfs.getCalendarByServiceId('WEEKDAY'); expect(calendar).toBeDefined(); expect(calendar?.service_id).toBe('WEEKDAY'); expect(calendar?.monday).toBe(1); }); - it('should return empty array for date with no service', () => { - const serviceIds = gtfs.getActiveServiceIds('21000101'); + it('should return empty array for date with no service', async () => { + const serviceIds = await gtfs.getActiveServiceIds('21000101'); expect(serviceIds.length).toBe(0); }); }); describe('Trip methods', () => { - it('should get trip by ID using filters', () => { - const trips = gtfs.getTrips({ tripId: 'TRIP1' }); + it('should get trip by ID using filters', async () => { + const trips = await gtfs.getTrips({ tripId: 'TRIP1' }); expect(trips.length).toBe(1); expect(trips[0].trip_id).toBe('TRIP1'); expect(trips[0].route_id).toBe('ROUTE1'); }); - it('should get trips by route', () => { - const trips = gtfs.getTrips({ routeId: 'ROUTE1' }); + it('should get trips by route', async () => { + const trips = await gtfs.getTrips({ routeId: 'ROUTE1' }); expect(trips.length).toBeGreaterThan(0); }); - it('should get trips by route and date', () => { - const trips = gtfs.getTrips({ routeId: 'ROUTE1', date: '20240101' }); + it('should get trips by route and date', async () => { + const trips = await gtfs.getTrips({ routeId: 'ROUTE1', date: '20240101' }); expect(trips.length).toBeGreaterThan(0); }); - it('should get trips by route, date, and direction', () => { - const trips = gtfs.getTrips({ routeId: 'ROUTE1', date: '20240101', directionId: 0 }); + it('should get trips by route, date, and direction', async () => { + const trips = await gtfs.getTrips({ routeId: 'ROUTE1', date: '20240101', directionId: 0 }); expect(trips.length).toBeGreaterThan(0); expect(trips.every(t => t.direction_id === 0)).toBe(true); }); - it('should return empty array for out-of-range date', () => { - const trips = gtfs.getTrips({ date: '19700101' }); + it('should return empty array for out-of-range date', async () => { + const trips = await gtfs.getTrips({ date: '19700101' }); expect(trips).toEqual([]); }); - it('should get multiple trips by ID array', () => { - const trips = gtfs.getTrips({ tripId: ['TRIP1', 'TRIP2'] }); + it('should get multiple trips by ID array', async () => { + const trips = await gtfs.getTrips({ tripId: ['TRIP1', 'TRIP2'] }); expect(trips.length).toBeGreaterThanOrEqual(1); }); }); describe('Stop time methods', () => { - it('should get stop times by trip using filters', () => { - const stopTimes = gtfs.getStopTimes({ tripId: 'TRIP1' }); + it('should get stop times by trip using filters', async () => { + const stopTimes = await gtfs.getStopTimes({ tripId: 'TRIP1' }); expect(stopTimes.length).toBeGreaterThan(0); expect(stopTimes[0].trip_id).toBe('TRIP1'); }); - it('should get stop times by stop', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'STOP1' }); + it('should get stop times by stop', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP1' }); expect(stopTimes.length).toBeGreaterThan(0); }); - it('should get stop times for stop, route, and date', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'STOP1', routeId: 'ROUTE1', date: '20240101' }); + it('should get stop times for stop, route, and date', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP1', routeId: 'ROUTE1', date: '20240101' }); expect(stopTimes.length).toBeGreaterThan(0); }); - it('should get stop times with direction filter', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'STOP1', routeId: 'ROUTE1', date: '20240101', directionId: 0 }); + it('should get stop times with direction filter', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'STOP1', routeId: 'ROUTE1', date: '20240101', directionId: 0 }); expect(stopTimes.length).toBeGreaterThan(0); }); - it('should return empty array for out-of-range date', () => { - const stopTimes = gtfs.getStopTimes({ date: '19700101' }); + it('should return empty array for out-of-range date', async () => { + const stopTimes = await gtfs.getStopTimes({ date: '19700101' }); expect(stopTimes).toEqual([]); }); - it('should get multiple stop times by trip ID array', () => { - const stopTimes = gtfs.getStopTimes({ tripId: ['TRIP1', 'TRIP2'] }); + it('should get multiple stop times by trip ID array', async () => { + const stopTimes = await gtfs.getStopTimes({ tripId: ['TRIP1', 'TRIP2'] }); expect(stopTimes.length).toBeGreaterThan(0); }); }); describe('Database export', () => { - it('should export database to ArrayBuffer', () => { - const buffer = gtfs.export(); + it('should export database to ArrayBuffer', async () => { + const buffer = await gtfs.export(); expect(buffer).toBeInstanceOf(ArrayBuffer); expect(buffer.byteLength).toBeGreaterThan(0); }); it('should be able to reload exported database', async () => { - const buffer = gtfs.export(); - const newGtfs = await GtfsSqlJs.fromDatabase(buffer, { SQL }); + const buffer = await gtfs.export(); + const newGtfs = await GtfsSqlJs.fromDatabase(buffer, { + adapter: await createSqlJsAdapter({ SQL }), + }); - const stops = newGtfs.getStops({ stopId: 'STOP1' }); + const stops = await newGtfs.getStops({ stopId: 'STOP1' }); expect(stops.length).toBe(1); expect(stops[0].stop_name).toBe('First Street'); - newGtfs.close(); + await newGtfs.close(); }); }); describe('Error handling', () => { - it('should throw error when accessing closed database', () => { + it('should throw error when accessing closed database', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any const closedGtfs = new (GtfsSqlJs as any)(); - expect(() => closedGtfs.getStops({ stopId: 'STOP1' })).toThrow('Database not initialized'); + await expect(closedGtfs.getStops({ stopId: 'STOP1' })).rejects.toThrow('Database not initialized'); }); }); }); diff --git a/tests/helpers/test-database.ts b/tests/helpers/test-database.ts index 730a00b..e23bd60 100644 --- a/tests/helpers/test-database.ts +++ b/tests/helpers/test-database.ts @@ -2,7 +2,7 @@ * Helper to create a test database with sample GTFS data */ -import type { Database, SqlJsStatic } from 'sql.js'; +import type { SqlJsStatic } from 'sql.js'; import { getAllCreateStatements } from '../../src/schema/schema'; export async function createTestDatabase(SQL: SqlJsStatic): Promise { diff --git a/tests/progress-callback.test.ts b/tests/progress-callback.test.ts index 98bb5b9..44a3f78 100644 --- a/tests/progress-callback.test.ts +++ b/tests/progress-callback.test.ts @@ -1,15 +1,10 @@ /** * Tests for the progress callback emitted during GTFS ingestion. - * - * Guards against regressions in: - * - totalRows estimate accuracy (within a small tolerance of the real - * COUNT(*) sum after ingestion completes) - * - percentComplete bounds (stays in [0, 100]) - * - the terminal 'complete' phase being emitted exactly once */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { GtfsSqlJs, type ProgressInfo } from '../src/gtfs-sqljs'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; import path from 'path'; import fs from 'fs/promises'; @@ -24,6 +19,7 @@ describe('Progress callback', () => { events = []; gtfs = await GtfsSqlJs.fromZipData(zipData, { + adapter: await createSqlJsAdapter(), onProgress: (info) => events.push({ ...info }), }); @@ -48,16 +44,17 @@ describe('Progress callback', () => { ]; let total = 0; for (const t of tables) { - const stmt = db.prepare(`SELECT COUNT(*) AS n FROM ${t}`); - stmt.step(); - total += (stmt.getAsObject().n as number) || 0; - stmt.free(); + const stmt = await db.prepare(`SELECT COUNT(*) AS n FROM ${t}`); + await stmt.step(); + const row = await stmt.getAsObject(); + total += (row.n as number) || 0; + await stmt.free(); } actualRowCount = total; }); - afterAll(() => { - gtfs?.close(); + afterAll(async () => { + await gtfs?.close(); }); it('emits at least one progress event', () => { @@ -65,8 +62,6 @@ describe('Progress callback', () => { }); it('reports a totalRows estimate close to the real COUNT(*) sum', () => { - // GtfsSqlJs emits one synthetic inserting_data event with totalRows=0 - // before loadGTFSData starts; ignore it and look at the loader's events. const loaderEvents = events.filter( (e) => e.phase === 'inserting_data' && e.totalRows > 0 ); @@ -76,8 +71,6 @@ describe('Progress callback', () => { expect(estimates.size).toBe(1); const estimate = loaderEvents[0].totalRows; - // Tolerance: up to one row of drift per GTFS file to absorb trailing - // blank lines or missing trailing newlines. The fixture has <=20 files. expect(Math.abs(estimate - actualRowCount)).toBeLessThanOrEqual(20); }); diff --git a/tests/sample-feed.test.ts b/tests/sample-feed.test.ts index 118b0d7..ff73451 100644 --- a/tests/sample-feed.test.ts +++ b/tests/sample-feed.test.ts @@ -9,6 +9,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { GtfsSqlJs } from '../src/gtfs-sqljs'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; import path from 'path'; import fs from 'fs/promises'; @@ -19,23 +20,21 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { // Load the sample GTFS feed const feedPath = path.join(__dirname, 'fixtures', 'sample-feed.zip'); const zipData = await fs.readFile(feedPath); - gtfs = await GtfsSqlJs.fromZipData(zipData); + gtfs = await GtfsSqlJs.fromZipData(zipData, { + adapter: await createSqlJsAdapter(), + }); }); - afterAll(() => { - gtfs?.close(); + afterAll(async () => { + await gtfs?.close(); }); describe('Agency', () => { - it('should have DTA (Demo Transit Authority)', () => { - const db = gtfs.getDatabase(); - const stmt = db.prepare('SELECT * FROM agency WHERE agency_id = ?'); - stmt.bind(['DTA']); - - expect(stmt.step()).toBe(true); - const agency = stmt.getAsObject(); - stmt.free(); + it('should have DTA (Demo Transit Authority)', async () => { + const agencies = await gtfs.getAgencies({ agencyId: 'DTA' }); + expect(agencies.length).toBe(1); + const agency = agencies[0]; expect(agency.agency_id).toBe('DTA'); expect(agency.agency_name).toBe('Demo Transit Authority'); expect(agency.agency_url).toBe('http://google.com'); @@ -44,13 +43,13 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { }); describe('Stops', () => { - it('should have 9 stops', () => { - const stops = gtfs.getStops(); + it('should have 9 stops', async () => { + const stops = await gtfs.getStops(); expect(stops.length).toBe(9); }); - it('should have BEATTY_AIRPORT with correct details', () => { - const stop = gtfs.getStops({ stopId: 'BEATTY_AIRPORT' })[0]; + it('should have BEATTY_AIRPORT with correct details', async () => { + const stop = (await gtfs.getStops({ stopId: 'BEATTY_AIRPORT' }))[0]; expect(stop).not.toBeNull(); expect(stop!.stop_id).toBe('BEATTY_AIRPORT'); @@ -59,8 +58,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(stop!.stop_lon).toBeCloseTo(-116.784582, 5); }); - it('should have BULLFROG with correct details', () => { - const stop = gtfs.getStops({ stopId: 'BULLFROG' })[0]; + it('should have BULLFROG with correct details', async () => { + const stop = (await gtfs.getStops({ stopId: 'BULLFROG' }))[0]; expect(stop).not.toBeNull(); expect(stop!.stop_id).toBe('BULLFROG'); @@ -69,8 +68,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(stop!.stop_lon).toBeCloseTo(-116.81797, 5); }); - it('should find stops when searching for "Airport"', () => { - const stops = gtfs.getStops({ name: 'Airport' }); + it('should find stops when searching for "Airport"', async () => { + const stops = await gtfs.getStops({ name: 'Airport' }); expect(stops.length).toBeGreaterThanOrEqual(1); const airportStop = stops.find(s => s.stop_id === 'BEATTY_AIRPORT'); @@ -78,8 +77,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(airportStop!.stop_name).toBe('Nye County Airport (Demo)'); }); - it('should get stops for trip AB1 in correct order', () => { - const stops = gtfs.getStops({ tripId: 'AB1' }); + it('should get stops for trip AB1 in correct order', async () => { + const stops = await gtfs.getStops({ tripId: 'AB1' }); expect(stops.length).toBe(2); expect(stops[0].stop_id).toBe('BEATTY_AIRPORT'); @@ -88,8 +87,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(stops[1].stop_name).toBe('Bullfrog (Demo)'); }); - it('should get stops for trip CITY1 in correct order', () => { - const stops = gtfs.getStops({ tripId: 'CITY1' }); + it('should get stops for trip CITY1 in correct order', async () => { + const stops = await gtfs.getStops({ tripId: 'CITY1' }); expect(stops.length).toBe(5); expect(stops[0].stop_id).toBe('STAGECOACH'); @@ -101,13 +100,13 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { }); describe('Routes', () => { - it('should have 5 routes', () => { - const routes = gtfs.getRoutes(); + it('should have 5 routes', async () => { + const routes = await gtfs.getRoutes(); expect(routes.length).toBe(5); }); - it('should have route AB (Airport - Bullfrog)', () => { - const route = gtfs.getRoutes({ routeId: 'AB' })[0]; + it('should have route AB (Airport - Bullfrog)', async () => { + const route = (await gtfs.getRoutes({ routeId: 'AB' }))[0]; expect(route).not.toBeNull(); expect(route!.route_id).toBe('AB'); @@ -117,8 +116,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(route!.route_type).toBe(3); // Bus }); - it('should have route BFC (Bullfrog - Furnace Creek Resort)', () => { - const route = gtfs.getRoutes({ routeId: 'BFC' })[0]; + it('should have route BFC (Bullfrog - Furnace Creek Resort)', async () => { + const route = (await gtfs.getRoutes({ routeId: 'BFC' }))[0]; expect(route).not.toBeNull(); expect(route!.route_id).toBe('BFC'); @@ -127,8 +126,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(route!.route_type).toBe(3); }); - it('should have route CITY (City)', () => { - const route = gtfs.getRoutes({ routeId: 'CITY' })[0]; + it('should have route CITY (City)', async () => { + const route = (await gtfs.getRoutes({ routeId: 'CITY' }))[0]; expect(route).not.toBeNull(); expect(route!.route_id).toBe('CITY'); @@ -136,15 +135,15 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(route!.route_long_name).toBe('City'); }); - it('should get all routes for agency DTA', () => { - const routes = gtfs.getRoutes({ agencyId: 'DTA' }); + it('should get all routes for agency DTA', async () => { + const routes = await gtfs.getRoutes({ agencyId: 'DTA' }); expect(routes.length).toBe(5); }); }); describe('Calendar', () => { - it('should have FULLW service (full week)', () => { - const calendar = gtfs.getCalendarByServiceId('FULLW'); + it('should have FULLW service (full week)', async () => { + const calendar = await gtfs.getCalendarByServiceId('FULLW'); expect(calendar).not.toBeNull(); expect(calendar!.service_id).toBe('FULLW'); @@ -159,8 +158,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(calendar!.end_date).toBe('20101231'); }); - it('should have WE service (weekend only)', () => { - const calendar = gtfs.getCalendarByServiceId('WE'); + it('should have WE service (weekend only)', async () => { + const calendar = await gtfs.getCalendarByServiceId('WE'); expect(calendar).not.toBeNull(); expect(calendar!.service_id).toBe('WE'); @@ -175,39 +174,39 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(calendar!.end_date).toBe('20101231'); }); - it('should return FULLW service for Monday 2007-01-01', () => { + it('should return FULLW service for Monday 2007-01-01', async () => { // 2007-01-01 was a Monday - const serviceIds = gtfs.getActiveServiceIds('20070101'); + const serviceIds = await gtfs.getActiveServiceIds('20070101'); expect(serviceIds).toContain('FULLW'); expect(serviceIds).not.toContain('WE'); // Weekend service not active on Monday }); - it('should return both FULLW and WE services for Saturday 2007-01-06', () => { + it('should return both FULLW and WE services for Saturday 2007-01-06', async () => { // 2007-01-06 was a Saturday - const serviceIds = gtfs.getActiveServiceIds('20070106'); + const serviceIds = await gtfs.getActiveServiceIds('20070106'); expect(serviceIds).toContain('FULLW'); expect(serviceIds).toContain('WE'); }); - it('should return both services for Sunday 2007-01-07', () => { + it('should return both services for Sunday 2007-01-07', async () => { // 2007-01-07 was a Sunday - const serviceIds = gtfs.getActiveServiceIds('20070107'); + const serviceIds = await gtfs.getActiveServiceIds('20070107'); expect(serviceIds).toContain('FULLW'); expect(serviceIds).toContain('WE'); }); - it('should return no services for date outside range', () => { - const serviceIds = gtfs.getActiveServiceIds('20110101'); + it('should return no services for date outside range', async () => { + const serviceIds = await gtfs.getActiveServiceIds('20110101'); expect(serviceIds.length).toBe(0); }); }); describe('Trips', () => { - it('should have trip AB1 (to Bullfrog)', () => { - const trip = gtfs.getTrips({ tripId: 'AB1' })[0]; + it('should have trip AB1 (to Bullfrog)', async () => { + const trip = (await gtfs.getTrips({ tripId: 'AB1' }))[0]; expect(trip).not.toBeNull(); expect(trip!.trip_id).toBe('AB1'); @@ -218,8 +217,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(trip!.block_id).toBe('1'); }); - it('should have trip AB2 (to Airport)', () => { - const trip = gtfs.getTrips({ tripId: 'AB2' })[0]; + it('should have trip AB2 (to Airport)', async () => { + const trip = (await gtfs.getTrips({ tripId: 'AB2' }))[0]; expect(trip).not.toBeNull(); expect(trip!.trip_id).toBe('AB2'); @@ -230,28 +229,28 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(trip!.block_id).toBe('2'); }); - it('should get 2 trips for route AB', () => { - const trips = gtfs.getTrips({ routeId: 'AB' }); + it('should get 2 trips for route AB', async () => { + const trips = await gtfs.getTrips({ routeId: 'AB' }); expect(trips.length).toBe(2); const tripIds = trips.map(t => t.trip_id).sort(); expect(tripIds).toEqual(['AB1', 'AB2']); }); - it('should get trips for route AB on Monday', () => { - const trips = gtfs.getTrips({ routeId: 'AB', date: '20070101' }); + it('should get trips for route AB on Monday', async () => { + const trips = await gtfs.getTrips({ routeId: 'AB', date: '20070101' }); expect(trips.length).toBe(2); expect(trips.every(t => t.service_id === 'FULLW')).toBe(true); }); - it('should get 7 trips for Monday (all FULLW trips)', () => { - const trips = gtfs.getTrips({ date: '20070101' }); - expect(trips.length).toBe(7); // AB1, AB2, STBA, CITY1, CITY2, BFC1, BFC2 + it('should get 7 trips for Monday (all FULLW trips)', async () => { + const trips = await gtfs.getTrips({ date: '20070101' }); + expect(trips.length).toBe(7); expect(trips.every(t => t.service_id === 'FULLW')).toBe(true); }); - it('should get 11 trips for Saturday (FULLW + WE trips)', () => { - const trips = gtfs.getTrips({ date: '20070106' }); + it('should get 11 trips for Saturday (FULLW + WE trips)', async () => { + const trips = await gtfs.getTrips({ date: '20070106' }); expect(trips.length).toBe(11); // 7 FULLW + 4 AAMV (WE) const fullwTrips = trips.filter(t => t.service_id === 'FULLW'); @@ -261,9 +260,9 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(weTrips.length).toBe(4); }); - it('should get trips by direction', () => { - const tripsDir0 = gtfs.getTrips({ routeId: 'AB', date: '20070101', directionId: 0 }); - const tripsDir1 = gtfs.getTrips({ routeId: 'AB', date: '20070101', directionId: 1 }); + it('should get trips by direction', async () => { + const tripsDir0 = await gtfs.getTrips({ routeId: 'AB', date: '20070101', directionId: 0 }); + const tripsDir1 = await gtfs.getTrips({ routeId: 'AB', date: '20070101', directionId: 1 }); expect(tripsDir0.length).toBe(1); expect(tripsDir0[0].trip_id).toBe('AB1'); @@ -276,8 +275,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { }); describe('Stop Times', () => { - it('should have correct stop times for trip AB1', () => { - const stopTimes = gtfs.getStopTimes({ tripId: 'AB1' }); + it('should have correct stop times for trip AB1', async () => { + const stopTimes = await gtfs.getStopTimes({ tripId: 'AB1' }); expect(stopTimes.length).toBe(2); @@ -296,8 +295,8 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(stopTimes[1].stop_sequence).toBe(2); }); - it('should have correct stop times for trip CITY1', () => { - const stopTimes = gtfs.getStopTimes({ tripId: 'CITY1' }); + it('should have correct stop times for trip CITY1', async () => { + const stopTimes = await gtfs.getStopTimes({ tripId: 'CITY1' }); expect(stopTimes.length).toBe(5); @@ -314,26 +313,23 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(stopTimes[4].stop_id).toBe('EMSI'); }); - it('should get stop times for BEATTY_AIRPORT', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', limit: 100 }); + it('should get stop times for BEATTY_AIRPORT', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', limit: 100 }); // BEATTY_AIRPORT appears in: STBA, AB1, AB2, AAMV1, AAMV2, AAMV3, AAMV4 = 7 trips expect(stopTimes.length).toBe(7); - // Should be ordered by arrival_time (as strings) - // Note: SQL ORDER BY treats times as strings, so "11:00:00" < "6:20:00" alphabetically const times = stopTimes.map(st => st.arrival_time); expect(times).toContain('6:20:00'); // STBA expect(times).toContain('8:00:00'); // AB1 or AAMV1 expect(times).toContain('12:15:00'); // AB2 }); - it('should get stop times for route AB at BEATTY_AIRPORT on Monday', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', routeId: 'AB', date: '20070101' }); + it('should get stop times for route AB at BEATTY_AIRPORT on Monday', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', routeId: 'AB', date: '20070101' }); expect(stopTimes.length).toBe(2); // AB1 and AB2 - // Check specific times const ab1Time = stopTimes.find(st => st.trip_id === 'AB1'); const ab2Time = stopTimes.find(st => st.trip_id === 'AB2'); @@ -344,108 +340,108 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(ab2Time!.arrival_time).toBe('12:15:00'); }); - it('should filter by direction', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', routeId: 'AB', date: '20070101', directionId: 0 }); + it('should filter by direction', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'BEATTY_AIRPORT', routeId: 'AB', date: '20070101', directionId: 0 }); expect(stopTimes.length).toBe(1); expect(stopTimes[0].trip_id).toBe('AB1'); expect(stopTimes[0].arrival_time).toBe('8:00:00'); }); - it('should filter by pickupType', () => { + it('should filter by pickupType', async () => { const db = gtfs.getDatabase(); // Set pickup_type=1 (no pickup) for AB1 at BEATTY_AIRPORT - db.run("UPDATE stop_times SET pickup_type = 1 WHERE trip_id = 'AB1' AND stop_id = 'BEATTY_AIRPORT'"); + await db.run("UPDATE stop_times SET pickup_type = 1 WHERE trip_id = 'AB1' AND stop_id = 'BEATTY_AIRPORT'"); // Set pickup_type=0 (regular) for AB1 at BULLFROG - db.run("UPDATE stop_times SET pickup_type = 0 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); + await db.run("UPDATE stop_times SET pickup_type = 0 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); - const regularOnly = gtfs.getStopTimes({ tripId: 'AB1', pickupType: 0 }); + const regularOnly = await gtfs.getStopTimes({ tripId: 'AB1', pickupType: 0 }); expect(regularOnly.length).toBe(1); expect(regularOnly[0].stop_id).toBe('BULLFROG'); - const noPickup = gtfs.getStopTimes({ tripId: 'AB1', pickupType: 1 }); + const noPickup = await gtfs.getStopTimes({ tripId: 'AB1', pickupType: 1 }); expect(noPickup.length).toBe(1); expect(noPickup[0].stop_id).toBe('BEATTY_AIRPORT'); // Array filter: both types - const both = gtfs.getStopTimes({ tripId: 'AB1', pickupType: [0, 1] }); + const both = await gtfs.getStopTimes({ tripId: 'AB1', pickupType: [0, 1] }); expect(both.length).toBe(2); // Reset - db.run("UPDATE stop_times SET pickup_type = NULL WHERE trip_id = 'AB1'"); + await db.run("UPDATE stop_times SET pickup_type = NULL WHERE trip_id = 'AB1'"); }); - it('should treat NULL pickup_type as REGULAR (0) per GTFS spec', () => { + it('should treat NULL pickup_type as REGULAR (0) per GTFS spec', async () => { // Sample feed has NULL pickup_type for all rows — GTFS spec says empty = 0 - const allAB1 = gtfs.getStopTimes({ tripId: 'AB1' }); + const allAB1 = await gtfs.getStopTimes({ tripId: 'AB1' }); expect(allAB1.length).toBe(2); // Filtering for REGULAR should match NULL rows - const regular = gtfs.getStopTimes({ tripId: 'AB1', pickupType: 0 }); + const regular = await gtfs.getStopTimes({ tripId: 'AB1', pickupType: 0 }); expect(regular.length).toBe(2); // Filtering for NONE should exclude NULL rows - const none = gtfs.getStopTimes({ tripId: 'AB1', pickupType: 1 }); + const none = await gtfs.getStopTimes({ tripId: 'AB1', pickupType: 1 }); expect(none.length).toBe(0); }); - it('should treat NULL drop_off_type as REGULAR (0) per GTFS spec', () => { - const regular = gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 0 }); + it('should treat NULL drop_off_type as REGULAR (0) per GTFS spec', async () => { + const regular = await gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 0 }); expect(regular.length).toBe(2); - const none = gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 1 }); + const none = await gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 1 }); expect(none.length).toBe(0); }); - it('should filter by dropOffType', () => { + it('should filter by dropOffType', async () => { const db = gtfs.getDatabase(); // Set drop_off_type=1 (no drop-off) for AB1 at BEATTY_AIRPORT - db.run("UPDATE stop_times SET drop_off_type = 1 WHERE trip_id = 'AB1' AND stop_id = 'BEATTY_AIRPORT'"); + await db.run("UPDATE stop_times SET drop_off_type = 1 WHERE trip_id = 'AB1' AND stop_id = 'BEATTY_AIRPORT'"); // Set drop_off_type=0 (regular) for AB1 at BULLFROG - db.run("UPDATE stop_times SET drop_off_type = 0 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); + await db.run("UPDATE stop_times SET drop_off_type = 0 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); - const regularOnly = gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 0 }); + const regularOnly = await gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 0 }); expect(regularOnly.length).toBe(1); expect(regularOnly[0].stop_id).toBe('BULLFROG'); - const noDropOff = gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 1 }); + const noDropOff = await gtfs.getStopTimes({ tripId: 'AB1', dropOffType: 1 }); expect(noDropOff.length).toBe(1); expect(noDropOff[0].stop_id).toBe('BEATTY_AIRPORT'); // Reset - db.run("UPDATE stop_times SET drop_off_type = NULL WHERE trip_id = 'AB1'"); + await db.run("UPDATE stop_times SET drop_off_type = NULL WHERE trip_id = 'AB1'"); }); - it('should filter by pickupType with trips join', () => { + it('should filter by pickupType with trips join', async () => { const db = gtfs.getDatabase(); - db.run("UPDATE stop_times SET pickup_type = 2 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); + await db.run("UPDATE stop_times SET pickup_type = 2 WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); // routeId forces a trips join - const results = gtfs.getStopTimes({ routeId: 'AB', pickupType: 2 }); + const results = await gtfs.getStopTimes({ routeId: 'AB', pickupType: 2 }); expect(results.length).toBe(1); expect(results[0].trip_id).toBe('AB1'); expect(results[0].stop_id).toBe('BULLFROG'); // Reset - db.run("UPDATE stop_times SET pickup_type = NULL WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); + await db.run("UPDATE stop_times SET pickup_type = NULL WHERE trip_id = 'AB1' AND stop_id = 'BULLFROG'"); }); }); describe('Complete Journey Scenarios', () => { - it('should plan journey from Airport to Bullfrog at 8am', () => { + it('should plan journey from Airport to Bullfrog at 8am', async () => { // User wants to go from BEATTY_AIRPORT to BULLFROG - const origin = gtfs.getStops({ stopId: 'BEATTY_AIRPORT' })[0]; - const destination = gtfs.getStops({ stopId: 'BULLFROG' })[0]; + const origin = (await gtfs.getStops({ stopId: 'BEATTY_AIRPORT' }))[0]; + const destination = (await gtfs.getStops({ stopId: 'BULLFROG' }))[0]; expect(origin).not.toBeNull(); expect(destination).not.toBeNull(); // Find route AB - const route = gtfs.getRoutes({ routeId: 'AB' })[0]; + const route = (await gtfs.getRoutes({ routeId: 'AB' }))[0]; expect(route).not.toBeNull(); // Get trips on a Monday - const trips = gtfs.getTrips({ routeId: 'AB', date: '20070101' }); + const trips = await gtfs.getTrips({ routeId: 'AB', date: '20070101' }); // Trip AB1 goes to Bullfrog (direction 0) const trip = trips.find(t => t.trip_headsign === 'to Bullfrog'); @@ -453,23 +449,15 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(trip!.trip_id).toBe('AB1'); // Get stop times - const stopTimes = gtfs.getStopTimes({ tripId: 'AB1' }); + const stopTimes = await gtfs.getStopTimes({ tripId: 'AB1' }); expect(stopTimes[0].stop_id).toBe('BEATTY_AIRPORT'); expect(stopTimes[0].departure_time).toBe('8:00:00'); expect(stopTimes[1].stop_id).toBe('BULLFROG'); expect(stopTimes[1].arrival_time).toBe('8:10:00'); - - console.log('\nJourney Plan:'); - console.log(`From: ${origin!.stop_name}`); - console.log(`To: ${destination!.stop_name}`); - console.log(`Route: ${route!.route_short_name} - ${route!.route_long_name}`); - console.log(`Depart: ${stopTimes[0].departure_time}`); - console.log(`Arrive: ${stopTimes[1].arrival_time}`); - console.log(`Duration: 10 minutes`); }); - it('should find all routes serving BULLFROG', () => { - const stopTimes = gtfs.getStopTimes({ stopId: 'BULLFROG', limit: 100 }); + it('should find all routes serving BULLFROG', async () => { + const stopTimes = await gtfs.getStopTimes({ stopId: 'BULLFROG', limit: 100 }); // Get unique trip IDs const tripIds = [...new Set(stopTimes.map(st => st.trip_id))]; @@ -477,7 +465,7 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { // Get routes for these trips const routes = new Set(); for (const tripId of tripIds) { - const trips = gtfs.getTrips({ tripId }); + const trips = await gtfs.getTrips({ tripId }); if (trips.length > 0) { routes.add(trips[0].route_id); } @@ -489,16 +477,12 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { expect(routes.has('BFC')).toBe(true); }); - it('should find city circuit route with all stops', () => { - const route = gtfs.getRoutes({ routeId: 'CITY' })[0]; + it('should find city circuit route with all stops', async () => { + const route = (await gtfs.getRoutes({ routeId: 'CITY' }))[0]; expect(route).not.toBeNull(); - // Get both directions - const trip1 = gtfs.getTrips({ tripId: 'CITY1' })[0]; - const trip2 = gtfs.getTrips({ tripId: 'CITY2' })[0]; - - const stops1 = gtfs.getStops({ tripId: 'CITY1' }); - const stops2 = gtfs.getStops({ tripId: 'CITY2' }); + const stops1 = await gtfs.getStops({ tripId: 'CITY1' }); + const stops2 = await gtfs.getStops({ tripId: 'CITY2' }); // Both trips have 5 stops expect(stops1.length).toBe(5); @@ -516,32 +500,30 @@ describe('Sample GTFS Feed Tests - Actual Data', () => { describe('Database Export/Import', () => { it('should export and re-import database with all data intact', async () => { // Export database - const buffer = gtfs.export(); + const buffer = await gtfs.export(); expect(buffer.byteLength).toBeGreaterThan(0); // Create new instance from exported buffer - const gtfs2 = await GtfsSqlJs.fromDatabase(buffer); + const gtfs2 = await GtfsSqlJs.fromDatabase(buffer, { + adapter: await createSqlJsAdapter(), + }); - // Verify specific data is intact - const db = gtfs2.getDatabase(); - const agency = db.prepare('SELECT * FROM agency WHERE agency_id = ?'); - agency.bind(['DTA']); - expect(agency.step()).toBe(true); - const agencyData = agency.getAsObject(); - agency.free(); - expect(agencyData.agency_name).toBe('Demo Transit Authority'); + // Verify agency via public API + const agencies = await gtfs2.getAgencies({ agencyId: 'DTA' }); + expect(agencies.length).toBe(1); + expect(agencies[0].agency_name).toBe('Demo Transit Authority'); // Verify route - const route = gtfs2.getRoutes({ routeId: 'AB' })[0]; + const route = (await gtfs2.getRoutes({ routeId: 'AB' }))[0]; expect(route).not.toBeNull(); expect(route!.route_long_name).toBe('Airport - Bullfrog'); // Verify trip - const trip = gtfs2.getTrips({ tripId: 'AB1' })[0]; + const trip = (await gtfs2.getTrips({ tripId: 'AB1' }))[0]; expect(trip).not.toBeNull(); expect(trip!.trip_headsign).toBe('to Bullfrog'); - gtfs2.close(); + await gtfs2.close(); }); }); }); diff --git a/tests/shapes.test.ts b/tests/shapes.test.ts index 610174b..88ddbcf 100644 --- a/tests/shapes.test.ts +++ b/tests/shapes.test.ts @@ -3,43 +3,46 @@ */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import initSqlJs from 'sql.js'; +import initSqlJs, { type SqlJsStatic } from 'sql.js'; import { GtfsSqlJs } from '../src/gtfs-sqljs'; +import { createSqlJsAdapter } from '../src/adapters/sql-js'; import { createTestDatabase } from './helpers/test-database'; describe('Shape methods', () => { let gtfs: GtfsSqlJs; - let SQL: any; + let SQL: SqlJsStatic; beforeAll(async () => { SQL = await initSqlJs(); const dbBuffer = await createTestDatabase(SQL); - gtfs = await GtfsSqlJs.fromDatabase(dbBuffer, { SQL }); + gtfs = await GtfsSqlJs.fromDatabase(dbBuffer, { + adapter: await createSqlJsAdapter({ SQL }), + }); }); - afterAll(() => { - gtfs?.close(); + afterAll(async () => { + await gtfs?.close(); }); describe('getShapes', () => { - it('should get all shapes', () => { - const shapes = gtfs.getShapes(); + it('should get all shapes', async () => { + const shapes = await gtfs.getShapes(); expect(shapes.length).toBe(7); // 5 points for SHAPE1 + 2 points for SHAPE2 }); - it('should get shapes by shape ID', () => { - const shapes = gtfs.getShapes({ shapeId: 'SHAPE1' }); + it('should get shapes by shape ID', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'SHAPE1' }); expect(shapes.length).toBe(5); expect(shapes.every(s => s.shape_id === 'SHAPE1')).toBe(true); }); - it('should get shapes by multiple shape IDs', () => { - const shapes = gtfs.getShapes({ shapeId: ['SHAPE1', 'SHAPE2'] }); + it('should get shapes by multiple shape IDs', async () => { + const shapes = await gtfs.getShapes({ shapeId: ['SHAPE1', 'SHAPE2'] }); expect(shapes.length).toBe(7); }); - it('should return shapes ordered by shape_id and shape_pt_sequence', () => { - const shapes = gtfs.getShapes({ shapeId: 'SHAPE1' }); + it('should return shapes ordered by shape_id and shape_pt_sequence', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'SHAPE1' }); expect(shapes.length).toBe(5); for (let i = 0; i < shapes.length; i++) { @@ -47,64 +50,64 @@ describe('Shape methods', () => { } }); - it('should get shapes by route ID', () => { - const shapes = gtfs.getShapes({ routeId: 'ROUTE1' }); + it('should get shapes by route ID', async () => { + const shapes = await gtfs.getShapes({ routeId: 'ROUTE1' }); expect(shapes.length).toBe(5); // SHAPE1 has 5 points, used by ROUTE1 expect(shapes.every(s => s.shape_id === 'SHAPE1')).toBe(true); }); - it('should get shapes by trip ID', () => { - const shapes = gtfs.getShapes({ tripId: 'TRIP4' }); + it('should get shapes by trip ID', async () => { + const shapes = await gtfs.getShapes({ tripId: 'TRIP4' }); expect(shapes.length).toBe(2); // SHAPE2 has 2 points, used by TRIP4 expect(shapes.every(s => s.shape_id === 'SHAPE2')).toBe(true); }); - it('should get shapes by multiple trip IDs', () => { - const shapes = gtfs.getShapes({ tripId: ['TRIP1', 'TRIP4'] }); + it('should get shapes by multiple trip IDs', async () => { + const shapes = await gtfs.getShapes({ tripId: ['TRIP1', 'TRIP4'] }); expect(shapes.length).toBe(7); // SHAPE1 (5) + SHAPE2 (2) }); - it('should include shape_dist_traveled when present', () => { - const shapes = gtfs.getShapes({ shapeId: 'SHAPE1' }); + it('should include shape_dist_traveled when present', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'SHAPE1' }); expect(shapes[0].shape_dist_traveled).toBe(0.0); expect(shapes[1].shape_dist_traveled).toBe(100.5); expect(shapes[4].shape_dist_traveled).toBe(400.0); }); - it('should have undefined shape_dist_traveled when not present', () => { - const shapes = gtfs.getShapes({ shapeId: 'SHAPE2' }); + it('should have undefined shape_dist_traveled when not present', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'SHAPE2' }); expect(shapes[0].shape_dist_traveled).toBeUndefined(); expect(shapes[1].shape_dist_traveled).toBeUndefined(); }); - it('should return empty array for non-existent shape ID', () => { - const shapes = gtfs.getShapes({ shapeId: 'NONEXISTENT' }); + it('should return empty array for non-existent shape ID', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'NONEXISTENT' }); expect(shapes.length).toBe(0); }); - it('should respect limit parameter', () => { - const shapes = gtfs.getShapes({ limit: 3 }); + it('should respect limit parameter', async () => { + const shapes = await gtfs.getShapes({ limit: 3 }); expect(shapes.length).toBe(3); }); - it('should have correct coordinate values', () => { - const shapes = gtfs.getShapes({ shapeId: 'SHAPE1' }); + it('should have correct coordinate values', async () => { + const shapes = await gtfs.getShapes({ shapeId: 'SHAPE1' }); expect(shapes[0].shape_pt_lat).toBeCloseTo(40.7128, 4); expect(shapes[0].shape_pt_lon).toBeCloseTo(-74.0060, 4); }); }); describe('getShapesToGeojson', () => { - it('should return a valid GeoJSON FeatureCollection', () => { - const geojson = gtfs.getShapesToGeojson(); + it('should return a valid GeoJSON FeatureCollection', async () => { + const geojson = await gtfs.getShapesToGeojson(); expect(geojson.type).toBe('FeatureCollection'); expect(Array.isArray(geojson.features)).toBe(true); expect(geojson.features.length).toBe(2); // 2 unique shapes }); - it('should create LineString features for each shape', () => { - const geojson = gtfs.getShapesToGeojson(); + it('should create LineString features for each shape', async () => { + const geojson = await gtfs.getShapesToGeojson(); for (const feature of geojson.features) { expect(feature.type).toBe('Feature'); @@ -113,8 +116,8 @@ describe('Shape methods', () => { } }); - it('should have coordinates in [lon, lat] order per GeoJSON spec', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); + it('should have coordinates in [lon, lat] order per GeoJSON spec', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); const feature = geojson.features[0]; // First coordinate should be [lon, lat] @@ -122,15 +125,15 @@ describe('Shape methods', () => { expect(feature.geometry.coordinates[0][1]).toBeCloseTo(40.7128, 4); // lat }); - it('should include shape_id in properties', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); + it('should include shape_id in properties', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); expect(geojson.features.length).toBe(1); expect(geojson.features[0].properties.shape_id).toBe('SHAPE1'); }); - it('should include route properties from first matching route', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); + it('should include route properties from first matching route', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); const props = geojson.features[0].properties; expect(props.route_id).toBe('ROUTE1'); @@ -139,23 +142,23 @@ describe('Shape methods', () => { expect(props.route_type).toBe(3); }); - it('should filter by route ID', () => { - const geojson = gtfs.getShapesToGeojson({ routeId: 'ROUTE2' }); + it('should filter by route ID', async () => { + const geojson = await gtfs.getShapesToGeojson({ routeId: 'ROUTE2' }); expect(geojson.features.length).toBe(1); expect(geojson.features[0].properties.shape_id).toBe('SHAPE2'); expect(geojson.features[0].geometry.coordinates.length).toBe(2); }); - it('should filter by trip ID', () => { - const geojson = gtfs.getShapesToGeojson({ tripId: 'TRIP4' }); + it('should filter by trip ID', async () => { + const geojson = await gtfs.getShapesToGeojson({ tripId: 'TRIP4' }); expect(geojson.features.length).toBe(1); expect(geojson.features[0].properties.shape_id).toBe('SHAPE2'); }); - it('should apply default precision of 6 decimals', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); + it('should apply default precision of 6 decimals', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); const coord = geojson.features[0].geometry.coordinates[0]; // Check that coordinates have at most 6 decimal places @@ -169,8 +172,8 @@ describe('Shape methods', () => { expect(latDecimals).toBeLessThanOrEqual(6); }); - it('should apply custom precision', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }, 2); + it('should apply custom precision', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }, 2); const coord = geojson.features[0].geometry.coordinates[0]; // With precision 2, -74.0060 should become -74.01 and 40.7128 should become 40.71 @@ -178,15 +181,15 @@ describe('Shape methods', () => { expect(coord[1]).toBeCloseTo(40.71, 2); }); - it('should return empty FeatureCollection for non-existent filter', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'NONEXISTENT' }); + it('should return empty FeatureCollection for non-existent filter', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'NONEXISTENT' }); expect(geojson.type).toBe('FeatureCollection'); expect(geojson.features.length).toBe(0); }); - it('should preserve coordinate order (sorted by shape_pt_sequence)', () => { - const geojson = gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); + it('should preserve coordinate order (sorted by shape_pt_sequence)', async () => { + const geojson = await gtfs.getShapesToGeojson({ shapeId: 'SHAPE1' }); const coords = geojson.features[0].geometry.coordinates; // First point should be at sequence 1 @@ -198,8 +201,8 @@ describe('Shape methods', () => { expect(coords[4][1]).toBeCloseTo(40.7148, 4); }); - it('should handle multiple shapes in one request', () => { - const geojson = gtfs.getShapesToGeojson(); + it('should handle multiple shapes in one request', async () => { + const geojson = await gtfs.getShapesToGeojson(); expect(geojson.features.length).toBe(2); @@ -207,8 +210,8 @@ describe('Shape methods', () => { expect(shapeIds).toEqual(['SHAPE1', 'SHAPE2']); }); - it('should have correct coordinate count for each shape', () => { - const geojson = gtfs.getShapesToGeojson(); + it('should have correct coordinate count for each shape', async () => { + const geojson = await gtfs.getShapesToGeojson(); const shape1 = geojson.features.find(f => f.properties.shape_id === 'SHAPE1'); const shape2 = geojson.features.find(f => f.properties.shape_id === 'SHAPE2'); diff --git a/tests/test-mobility-database-feeds.ts b/tests/test-mobility-database-feeds.ts index c807d83..e75b703 100644 --- a/tests/test-mobility-database-feeds.ts +++ b/tests/test-mobility-database-feeds.ts @@ -12,6 +12,7 @@ import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { GtfsSqlJs } from '../src/index.js'; +import { createSqlJsAdapter } from '../src/adapters/sql-js/index.js'; // ── ANSI colors ────────────────────────────────────────────────────────────── @@ -110,6 +111,7 @@ async function ingestFeed(url: string, label: string): Promise { let gtfs: GtfsSqlJs | undefined; try { gtfs = await GtfsSqlJs.fromZip(url, { + adapter: await createSqlJsAdapter(), skipFiles: ['shapes.txt'], onProgress: (info) => { process.stdout.write(`\r ${dim(info.message)} ${dim(`(${info.percentComplete}%)`)}`); @@ -117,8 +119,8 @@ async function ingestFeed(url: string, label: string): Promise { }); process.stdout.write('\r\x1b[K'); // clear progress line - const routes = gtfs.getRoutes(); - const agencies = gtfs.getAgencies(); + const routes = await gtfs.getRoutes(); + const agencies = await gtfs.getAgencies(); result.routeCount = routes.length; result.agencyNames = agencies.map((a) => a.agency_name); result.success = true; @@ -126,7 +128,7 @@ async function ingestFeed(url: string, label: string): Promise { process.stdout.write('\r\x1b[K'); result.error = err instanceof Error ? err.message : String(err); } finally { - gtfs?.close(); + await gtfs?.close(); } return result; } diff --git a/tsup.config.ts b/tsup.config.ts index 77c987e..d897001 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -1,7 +1,11 @@ import { defineConfig } from 'tsup'; export default defineConfig({ - entry: ['src/index.ts'], + entry: { + 'index': 'src/index.ts', + 'adapters/sql-js/index': 'src/adapters/sql-js/index.ts', + 'adapters/better-sqlite3/index': 'src/adapters/better-sqlite3/index.ts', + }, format: ['esm'], dts: true, splitting: false,