diff --git a/demos/demo_compare_pd_ibis.py b/demos/demo_compare_pd_ibis.py new file mode 100644 index 0000000..e2ce712 --- /dev/null +++ b/demos/demo_compare_pd_ibis.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +""" +Compare Leanframe workflows: pandas-first vs strict-Ibis-until-output. + +This demo answers two practical questions in one place: + +1) How to work with Leanframe when starting from pandas vs staying in Ibis. +2) A runnable side-by-side example using a tiny nested dataset. + +Scope: +- Simple data model (two columns, one nested) for each table. +- Same operations in both approaches: + - data creation + - nesting + extraction (NestedHandler.prepare) + - join on extracted nested fields + - indexing (.set_index + .head) + - materialize to pandas only for final display +""" + +import sys +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import ibis +import pandas as pd +import pyarrow as pa + +import leanframe +from leanframe.core.frame import DataFrame +from leanframe.core.nested_handler import NestedHandler + + +def print_concept_comparison() -> None: + """Explain practical tradeoffs between pandas-first and Ibis-first workflows.""" + print("\n" + "=" * 80) + print("Leanframe Workflow Comparison: pandas-first vs strict-Ibis") + print("=" * 80) + + print("\n1) Data Creation") + print("- pandas-first:") + print(" Build pandas DataFrames first, then call session.DataFrame(pandas_df).") + print(" Best when your source is already local/in-memory or notebook-centric.") + print("- strict-Ibis:") + print(" Build Arrow/Ibis tables and read them via session.read_ibis(...).") + print(" Best for backend-native execution (BigQuery-style workflow).") + + print("\n2) Nested Columns") + print("- Both paths use NestedHandler identically once data is in Leanframe DataFrames.") + print("- Use handler.prepare(name) to flatten nested fields into underscore columns.") + print(" Example: profile.email -> profile_email") + + print("\n3) Joins") + print("- pandas-first:") + print(" Easy startup, but you materialize data locally early.") + print("- strict-Ibis:") + print(" Keep execution in backend expressions as long as possible.") + print(" Better alignment with warehouse engines and larger datasets.") + + print("\n4) Indexing") + print("- Indexing API is the same in both approaches: set_index(), iloc, loc, head/tail.") + print("- Indexing composes well after nested extraction and joins.") + + print("\n5) When to use to_pandas()") + print("- pandas-first:") + print(" You already started in pandas, so conversion happened at ingestion.") + print("- strict-Ibis:") + print(" Use to_pandas() only at the very end (display/export/debug sample).") + + print("\n6) What Leanframe already feels like") + print("- DataFrame: columns, dtypes, assign, set_index, head, tail, loc, iloc") + print("- Series: sum, mean, min, max, count, isin, dtype, to_list") + print("- NestedHandler: join() and prepare() provide the SQL/Ibis bridge") + print("- Current gap vs pandas: no full merge/groupby/query/drop/reset_index parity yet") + print("- Practical takeaway: write pandas-shaped code, but keep execution deferred in Ibis") + + +def build_pandas_inputs() -> tuple[pd.DataFrame, pd.DataFrame]: + """Create tiny nested inputs in pandas for the pandas-first path.""" + customers_pd = pd.DataFrame( + { + "customer_id": [1, 2, 3], + "profile": [ + {"email": "alice@example.com", "age": 30}, + {"email": "bob@example.com", "age": 25}, + {"email": "charlie@example.com", "age": 35}, + ], + } + ) + + orders_pd = pd.DataFrame( + { + "order_id": [101, 102, 103], + "shipping": [ + {"recipient": {"email": "alice@example.com"}}, + {"recipient": {"email": "bob@example.com"}}, + {"recipient": {"email": "alice@example.com"}}, + ], + } + ) + + return customers_pd, orders_pd + + +def build_ibis_inputs(backend: ibis.BaseBackend) -> tuple[DataFrame, DataFrame]: + """Create the same logical data in Arrow/Ibis for the strict-Ibis path.""" + customers_tbl = backend.create_table( + f"customers_cmp_{uuid.uuid4().hex[:8]}", + pa.Table.from_pydict( + { + "customer_id": [1, 2, 3], + "profile": [ + {"email": "alice@example.com", "age": 30}, + {"email": "bob@example.com", "age": 25}, + {"email": "charlie@example.com", "age": 35}, + ], + }, + schema=pa.schema( + [ + pa.field("customer_id", pa.int64()), + pa.field( + "profile", + pa.struct( + [pa.field("email", pa.string()), pa.field("age", pa.int64())] + ), + ), + ] + ), + ), + temp=True, + ) + + orders_tbl = backend.create_table( + f"orders_cmp_{uuid.uuid4().hex[:8]}", + pa.Table.from_pydict( + { + "order_id": [101, 102, 103], + "shipping": [ + {"recipient": {"email": "alice@example.com"}}, + {"recipient": {"email": "bob@example.com"}}, + {"recipient": {"email": "alice@example.com"}}, + ], + }, + schema=pa.schema( + [ + pa.field("order_id", pa.int64()), + pa.field( + "shipping", + pa.struct( + [ + pa.field( + "recipient", + pa.struct([pa.field("email", pa.string())]), + ) + ] + ), + ), + ] + ), + ), + temp=True, + ) + + return DataFrame(customers_tbl), DataFrame(orders_tbl) + + +def run_pipeline(label: str, customers_df: DataFrame, orders_df: DataFrame) -> None: + """Run the same nesting, join, and indexing pipeline for either input style.""" + print("\n" + "-" * 80) + print(f"Pipeline: {label}") + print("-" * 80) + + handler = NestedHandler() + handler.add("customers", customers_df) + handler.add("orders", orders_df) + + # NestedHandler already gives a pandas-like join wrapper over Ibis. + joined_df = handler.join( + tables={"c": "customers", "o": "orders"}, + on=[("c", "profile_email", "o", "shipping_recipient_email")], + how="inner", + ) + + print(f"Joined rows: {joined_df['order_id'].count()}") + print(f"Joined columns: {joined_df.columns.tolist()}") + + # Keep the workflow pandas-shaped: index first, then take head(). + by_age_desc = joined_df.set_index("profile_age", ascending=False) + top_rows = by_age_desc.head(5) + + print(f"Oldest matched customer age: {joined_df['profile_age'].max()}") + + # Final materialization for output. + result_pd = top_rows.to_pandas()[ + ["customer_id", "profile_email", "profile_age", "order_id"] + ] + print("\nFinal output (pandas, materialized at the end):") + print(result_pd) + + +def main() -> None: + print_concept_comparison() + + backend = ibis.duckdb.connect() + session = leanframe.Session(backend=backend) + + try: + # Approach A: pandas-first ingestion. + customers_pd, orders_pd = build_pandas_inputs() + customers_from_pd = session.DataFrame(customers_pd) + orders_from_pd = session.DataFrame(orders_pd) + run_pipeline("A) pandas-first", customers_from_pd, orders_from_pd) + + # Approach B: strict-Ibis ingestion (Arrow/Ibis until final output). + customers_ibis, orders_ibis = build_ibis_inputs(backend) + customers_from_ibis = session.read_ibis(customers_ibis.to_ibis()) + orders_from_ibis = session.read_ibis(orders_ibis.to_ibis()) + run_pipeline( + "B) strict-Ibis (pandas only at the end)", + customers_from_ibis, + orders_from_ibis, + ) + finally: + backend.disconnect() + + +if __name__ == "__main__": + main() diff --git a/demos/demo_flexible_joins.py b/demos/demo_flexible_joins.py index 74c599c..c20da57 100644 --- a/demos/demo_flexible_joins.py +++ b/demos/demo_flexible_joins.py @@ -39,11 +39,12 @@ import sys from pathlib import Path +import uuid sys.path.insert(0, str(Path(__file__).parent.parent)) import ibis -import pandas as pd +import pyarrow as pa import leanframe from leanframe.core.frame import DataFrame from leanframe.core.nested_handler import NestedHandler @@ -74,20 +75,36 @@ def main(): customers_df = create_customers_for_join() orders_df = create_orders_for_join() + # Keep data in Ibis/Arrow space and register everything on the demo backend. + customers_tbl = backend.create_table( + f"customers_demo_{uuid.uuid4().hex[:8]}", + customers_df.to_ibis().to_pyarrow(), + temp=True, + ) + orders_tbl = backend.create_table( + f"orders_demo_{uuid.uuid4().hex[:8]}", + orders_df.to_ibis().to_pyarrow(), + temp=True, + ) + # Add third table - products - products_pd = pd.DataFrame( - { - "product_id": [1, 2, 3], - "name": ["Widget", "Gadget", "Doohickey"], - "category": ["Electronics", "Electronics", "Hardware"], - "price": [29.99, 149.99, 9.99], - } + products_tbl = backend.create_table( + f"products_demo_{uuid.uuid4().hex[:8]}", + pa.Table.from_pydict( + { + "product_id": [1, 2, 3], + "name": ["Widget", "Gadget", "Doohickey"], + "category": ["Electronics", "Electronics", "Hardware"], + "price": [29.99, 149.99, 9.99], + } + ), + temp=True, ) - products_df = session.DataFrame(products_pd) + products_df = session.read_ibis(products_tbl) # Add to NestedHandler - nested.add("customers", session.DataFrame(customers_df.to_pandas())) - nested.add("orders", session.DataFrame(orders_df.to_pandas())) + nested.add("customers", session.read_ibis(customers_tbl)) + nested.add("orders", session.read_ibis(orders_tbl)) nested.add("products", products_df) print("\n✅ Added 3 tables:") @@ -154,13 +171,27 @@ def main(): print("=" * 70) print("\n🎯 Goal: customers ⋈ orders ⋈ products") - print(" Old methods: ❌ Can't do this!") - print(" New approach: ✅ Easy!") - # Modify orders to have product_id - orders_with_products_pd = orders_flat.to_pandas() - orders_with_products_pd["product_id"] = [1, 2, 1, 3, 2] # Match order IDs - orders_with_products = session.DataFrame(orders_with_products_pd) + # Modify orders to have product_id without materializing to pandas. + order_product_map_tbl = backend.create_table( + f"order_product_map_demo_{uuid.uuid4().hex[:8]}", + pa.Table.from_pydict( + { + "map_order_id": [101, 102, 103, 104, 105], + "product_id": [1, 2, 1, 3, 2], + } + ), + temp=True, + ) + orders_with_products = DataFrame( + orders_flat._data.join( + order_product_map_tbl, + predicates=[ + orders_flat._data.order_id == order_product_map_tbl.map_order_id + ], + how="inner", + ).drop(order_product_map_tbl.map_order_id) + ) print("\n1️⃣ First join: customers ⋈ orders") step1 = customers_flat._data.join( diff --git a/demos/demo_indexing_with_nest_nopd.py b/demos/demo_indexing_with_nest_nopd.py new file mode 100644 index 0000000..0e58346 --- /dev/null +++ b/demos/demo_indexing_with_nest_nopd.py @@ -0,0 +1,547 @@ +""" +Example: Using Indexing with Nested Data in Leanframe (No pandas ingestion) + +This demo mirrors demo_indexing_with_nested.py but avoids pandas for data creation +and ingestion. Data is created as Python dicts, converted to Arrow tables, and +registered on the Ibis backend. pandas is used only at the very end for display +via DataFrame.to_pandas(). +""" + +from datetime import datetime +import uuid + +import ibis +import pyarrow as pa +import leanframe + +# Import leanframe components +from leanframe.core.frame import DataFrame, DataFrameHandler +from leanframe.core.nested_handler import NestedHandler + + +def register_table( + session: leanframe.Session, + backend: ibis.BaseBackend, + prefix: str, + data: dict, + schema: pa.Schema | None = None, +) -> DataFrame: + """Register dict data as an Arrow/Ibis temp table and return a Leanframe DataFrame.""" + table_name = f"{prefix}_{uuid.uuid4().hex[:8]}" + pa_table = pa.Table.from_pydict(data, schema=schema) + table = backend.create_table(table_name, pa_table, temp=True) + return session.read_ibis(table) + + +def create_sample_nested_data(): + """Create sample data with nested structures for testing.""" + + # Sample customer data with nested profiles + customers_data = { + "customer_id": [1001, 1002, 1003, 1004, 1005], + "profile": [ + {"name": "Alice Johnson", "age": 34, "email": "alice@example.com"}, + {"name": "Bob Smith", "age": 28, "email": "bob@example.com"}, + {"name": "Carol Davis", "age": 45, "email": "carol@example.com"}, + {"name": "David Wilson", "age": 31, "email": "david@example.com"}, + {"name": "Eve Martinez", "age": 39, "email": "eve@example.com"}, + ], + "registration_date": [ + datetime(2024, 1, 15), + datetime(2024, 2, 20), + datetime(2023, 11, 5), + datetime(2024, 3, 10), + datetime(2023, 12, 18), + ], + } + + # Sample order data + orders_data = { + "order_id": [5001, 5002, 5003, 5004, 5005, 5006], + "customer_id": [1001, 1001, 1002, 1003, 1004, 1005], + "amount": [299.99, 149.50, 599.00, 89.99, 450.00, 199.99], + "order_date": [ + datetime(2024, 3, 1), + datetime(2024, 3, 15), + datetime(2024, 3, 5), + datetime(2024, 3, 20), + datetime(2024, 3, 12), + datetime(2024, 3, 8), + ], + "status": [ + "completed", + "completed", + "pending", + "completed", + "shipped", + "completed", + ], + } + + return customers_data, orders_data + + +def demo_basic_indexing(session: leanframe.Session, backend: ibis.BaseBackend): + """Demo 1: Basic indexing without nested data.""" + print("\n" + "=" * 70) + print("DEMO 1: Basic Indexing") + print("=" * 70) + + data = { + "id": [1, 2, 3, 4, 5], + "value": [10, 20, 30, 40, 50], + "timestamp": [ + datetime(2024, 1, 1), + datetime(2024, 1, 2), + datetime(2024, 1, 3), + datetime(2024, 1, 4), + datetime(2024, 1, 5), + ], + } + + df = register_table(session, backend, "idx_basic", data) + + print(f"\nOriginal DataFrame shape: {len(df.columns)} columns") + print(f"Columns: {df.columns.tolist()}") + + # Set index on timestamp + print("\nSetting index on 'timestamp' (ascending)...") + df_indexed = df.set_index("timestamp", ascending=True) + print(f"Index: {df_indexed.index}") + + # Use iloc + print("\nUsing .iloc for position-based access:") + print("\n First 2 rows (df.iloc[0:2]):") + first_2 = df_indexed.iloc[0:2] + print(first_2.to_pandas()) + + # Use head/tail + print("\nUsing .head() and .tail():") + print("\n First 3 rows (df.head(3)):") + print(df_indexed.head(3).to_pandas()) + + print("\n Last 2 rows (df.tail(2)):") + print(df_indexed.tail(2).to_pandas()) + + # Use loc + print("\nSetting index on 'id' for .loc access:") + df_by_id = df.set_index("id") + print("\n Get row where id=3 (df.loc[3]):") + print(df_by_id.loc[3].to_pandas()) + + print("\n Get range id=2:4 (df.loc[2:4]):") + print(df_by_id.loc[2:4].to_pandas()) + + +def demo_nested_data_with_indexing( + session: leanframe.Session, backend: ibis.BaseBackend +): + """Demo 2: Indexing with nested data extraction.""" + print("\n" + "=" * 70) + print("DEMO 2: Nested Data + Indexing") + print("=" * 70) + + customers_data, _ = create_sample_nested_data() + + profile_schema = pa.struct( + [ + pa.field("name", pa.string()), + pa.field("age", pa.int64()), + pa.field("email", pa.string()), + ] + ) + customers_schema = pa.schema( + [ + pa.field("customer_id", pa.int64()), + pa.field("profile", profile_schema), + pa.field("registration_date", pa.timestamp("us")), + ] + ) + + customers_df = register_table( + session, backend, "idx_customers", customers_data, customers_schema + ) + + print("\nOriginal nested DataFrame:") + print(f"Columns: {customers_df.columns.tolist()}") + + # Create handler to analyze nested structure + print("\nCreating DataFrameHandler to analyze nested structure...") + handler = DataFrameHandler(customers_df) + handler.show_structure() + + # Extract nested fields + print("\nExtracting nested fields...") + flat_df = handler.extract_nested_fields(verbose=False) + print(f"Flattened columns: {flat_df.columns.tolist()}") + + # Now apply indexing to the flattened DataFrame + print("\nSetting index on 'profile_age' (descending - oldest first)...") + by_age = flat_df.set_index("profile_age", ascending=False) + + print("\nOldest 3 customers (by_age.head(3)):") + print( + by_age.head(3).to_pandas()[ + ["customer_id", "profile_name", "profile_age", "profile_email"] + ] + ) + + print("\nYoungest 2 customers (by_age.tail(2)):") + print( + by_age.tail(2).to_pandas()[ + ["customer_id", "profile_name", "profile_age", "profile_email"] + ] + ) + + # Index by registration date + print("\nSetting index on 'registration_date' (newest first)...") + by_date = flat_df.set_index("registration_date", ascending=False) + + print("\nMost recent 3 registrations (by_date.iloc[0:3]):") + print( + by_date.iloc[0:3].to_pandas()[ + ["customer_id", "profile_name", "registration_date"] + ] + ) + + # Use .loc on customer_id + print("\nSetting index on 'customer_id' for .loc access...") + by_id = flat_df.set_index("customer_id") + + print("\nGet specific customers (by_id.loc[[1001, 1003]]):") + print( + by_id.loc[[1001, 1003]].to_pandas()[ + ["customer_id", "profile_name", "profile_email"] + ] + ) + + +def demo_joins_with_indexing(session: leanframe.Session, backend: ibis.BaseBackend): + """Demo 3: Combining NestedHandler joins with indexing.""" + print("\n" + "=" * 70) + print("DEMO 3: Joins + Indexing") + print("=" * 70) + + customers_data, orders_data = create_sample_nested_data() + + profile_schema = pa.struct( + [ + pa.field("name", pa.string()), + pa.field("age", pa.int64()), + pa.field("email", pa.string()), + ] + ) + customers_schema = pa.schema( + [ + pa.field("customer_id", pa.int64()), + pa.field("profile", profile_schema), + pa.field("registration_date", pa.timestamp("us")), + ] + ) + + orders_schema = pa.schema( + [ + pa.field("order_id", pa.int64()), + pa.field("customer_id", pa.int64()), + pa.field("amount", pa.float64()), + pa.field("order_date", pa.timestamp("us")), + pa.field("status", pa.string()), + ] + ) + + customers_df = register_table( + session, backend, "idx_join_customers", customers_data, customers_schema + ) + orders_df = register_table(session, backend, "idx_join_orders", orders_data, orders_schema) + + # Setup NestedHandler + print("\nSetting up NestedHandler...") + handler = NestedHandler() + handler.add("customers", customers_df) + handler.add("orders", orders_df) + + # Prepare customers (extract nested fields) + print("\nPreparing customers DataFrame (extracting nested fields)...") + customers_prep = handler.prepare("customers", verbose=False) + + # Add prepared version back + handler.add("customers_flat", customers_prep) + + # Perform join + print("\nJoining customers with orders...") + joined = handler.join( + tables={"c": "customers_flat", "o": "orders"}, + on=[("c", "customer_id", "o", "customer_id")], + how="inner", + ) + + print(f"Joined DataFrame columns: {len(joined.columns)}") + + # Apply indexing to joined result + print("\nSetting index on 'order_date' (most recent first)...") + by_date = joined.set_index("order_date", ascending=False) + + print("\nMost recent 3 orders (by_date.head(3)):") + result = by_date.head(3).to_pandas() + print(result[["order_id", "profile_name", "amount", "order_date", "status"]]) + + # Index by amount + print("\nSetting index on 'amount' (highest first)...") + by_amount = joined.set_index("amount", ascending=False) + + print("\nTop 3 highest value orders (by_amount.iloc[0:3]):") + result = by_amount.iloc[0:3].to_pandas() + print(result[["order_id", "profile_name", "profile_email", "amount", "order_date"]]) + + # Use .loc to filter by customer_id + print("\nSetting index on 'customer_id' for .loc filtering...") + by_customer = joined.set_index("customer_id") + + print("\nAll orders for customer 1001 (by_customer.loc[1001]):") + result = by_customer.loc[1001].to_pandas() + print(result[["order_id", "profile_name", "amount", "order_date"]]) + + +def demo_chaining_operations(session: leanframe.Session, backend: ibis.BaseBackend): + """Demo 4: Chaining indexing with other operations.""" + print("\n" + "=" * 70) + print("DEMO 4: Chaining Operations") + print("=" * 70) + + _, orders_data = create_sample_nested_data() + + orders_schema = pa.schema( + [ + pa.field("order_id", pa.int64()), + pa.field("customer_id", pa.int64()), + pa.field("amount", pa.float64()), + pa.field("order_date", pa.timestamp("us")), + pa.field("status", pa.string()), + ] + ) + orders_df = register_table(session, backend, "idx_chain_orders", orders_data, orders_schema) + + print("\nOriginal orders:") + print(orders_df.to_pandas()) + + # Chain: filter -> set index -> slice + print("\nChaining: filter completed orders, order by date, get top 2...") + + # Filter completed orders (using Ibis directly) + completed = orders_df._data.filter(orders_df._data.status == "completed") + completed_df = DataFrame(completed) + + # Set index and slice + by_date = completed_df.set_index("order_date", ascending=False) + recent_completed = by_date.iloc[0:2] + + print("\nMost recent 2 completed orders:") + print(recent_completed.to_pandas()) + + # Chain: order by amount, get top 3, then filter by customer + print("\nChaining: order by amount (desc), get top 3...") + by_amount = orders_df.set_index("amount", ascending=False) + top_3_value = by_amount.iloc[0:3] + + print("\nTop 3 highest value orders:") + print(top_3_value.to_pandas()) + + +def demo_error_cases(session: leanframe.Session, backend: ibis.BaseBackend): + """Demo 5: Common error cases and how to handle them.""" + print("\n" + "=" * 70) + print("DEMO 5: Error Handling") + print("=" * 70) + + data = {"id": [1, 2, 3], "value": [10, 20, 30]} + df = register_table(session, backend, "idx_errors", data) + + # Error 1: Using .iloc without setting index + print("\nError 1: Using .iloc without index...") + try: + df.iloc[0] + except ValueError as e: + print(f"Caught expected error: {e}") + + # Error 2: Using .loc without setting index + print("\nError 2: Using .loc without index...") + try: + df.loc[1] + except ValueError as e: + print(f"Caught expected error: {e}") + + # Error 3: Setting index on non-existent column + print("\nError 3: Setting index on non-existent column...") + try: + df.set_index("nonexistent") + except KeyError as e: + print(f"Caught expected error: {e}") + + # Success: Proper usage + print("\nProper usage: set index first...") + df_indexed = df.set_index("id") + print(f"Index set: {df_indexed.index}") + print("Now .iloc and .loc work correctly!") + result = df_indexed.loc[2] + print(result.to_pandas()) + + +def demo_multi_column_ordering( + session: leanframe.Session, backend: ibis.BaseBackend +): + """Demo 6: Multi-column composite ordering (like SQL ORDER BY col1, col2).""" + print("\n" + "=" * 70) + print("DEMO 6: Multi-Column Ordering") + print("=" * 70) + + # Create sample data with priority and timestamp + task_data = { + "task_id": [101, 102, 103, 104, 105, 106, 107, 108], + "priority": [1, 1, 2, 2, 3, 3, 1, 2], + "timestamp": [ + datetime(2024, 3, 1, 10, 0), + datetime(2024, 3, 1, 9, 0), + datetime(2024, 3, 1, 11, 0), + datetime(2024, 3, 1, 8, 0), + datetime(2024, 3, 1, 12, 0), + datetime(2024, 3, 1, 7, 0), + datetime(2024, 3, 1, 13, 0), + datetime(2024, 3, 1, 14, 0), + ], + "description": [ + "Task A", + "Task B", + "Task C", + "Task D", + "Task E", + "Task F", + "Task G", + "Task H", + ], + } + + task_df = register_table(session, backend, "idx_tasks", task_data) + + print("\nOriginal data (unordered):") + print(task_df.to_pandas()[["task_id", "priority", "timestamp", "description"]]) + + # Single-column ordering + print("\nSingle-column index on 'priority' (ascending):") + by_priority = task_df.set_index("priority") + print(by_priority.to_pandas()[["task_id", "priority", "timestamp", "description"]]) + print("Note: Within each priority level, order is not deterministic") + + # Multi-column ordering - priority ASC, timestamp ASC + print("\nMulti-column index: ['priority', 'timestamp'] (both ascending):") + by_priority_time = task_df.set_index(["priority", "timestamp"], ascending=True) + print( + by_priority_time.to_pandas()[ + ["task_id", "priority", "timestamp", "description"] + ] + ) + print("Result: Ordered by priority, then by earliest timestamp within each priority") + + # Multi-column with different directions + print("\nMulti-column index: ['priority', 'timestamp'] with [DESC, ASC]:") + by_priority_desc = task_df.set_index(["priority", "timestamp"], ascending=[False, True]) + print( + by_priority_desc.to_pandas()[ + ["task_id", "priority", "timestamp", "description"] + ] + ) + print("Result: Highest priority first, then earliest timestamp (priority queue)") + + # Use with iloc + print("\nGetting top 3 tasks with multi-column ordering:") + print(" (Highest priority first, earliest timestamp breaks ties)") + top_3 = by_priority_desc.iloc[0:3] + print(top_3.to_pandas()[["task_id", "priority", "timestamp", "description"]]) + + # Example with nested data + print("\nMulti-column ordering with nested fields:") + customers_data, _ = create_sample_nested_data() + + profile_schema = pa.struct( + [ + pa.field("name", pa.string()), + pa.field("age", pa.int64()), + pa.field("email", pa.string()), + ] + ) + customers_schema = pa.schema( + [ + pa.field("customer_id", pa.int64()), + pa.field("profile", profile_schema), + pa.field("registration_date", pa.timestamp("us")), + ] + ) + + customers_df = register_table( + session, backend, "idx_multi_customers", customers_data, customers_schema + ) + handler = DataFrameHandler(customers_df) + flat_df = handler.extract_nested_fields(verbose=False) + + # Order by age DESC, then registration_date ASC + by_age_date = flat_df.set_index( + ["profile_age", "registration_date"], ascending=[False, True] + ) + print("\nOrdered by age DESC (nested field), registration_date ASC (regular field):") + print( + by_age_date.to_pandas()[ + ["customer_id", "profile_name", "profile_age", "registration_date"] + ] + ) + print("\nSQL equivalent: ORDER BY profile_age DESC, registration_date ASC") + print("\nThis demonstrates ordering across different nesting levels:") + print("- 'profile_age' comes from nested 'profile' struct") + print("- 'registration_date' is a regular top-level column") + + # More complex example: order by regular column, then multiple nested fields + print("\nComplex multi-level ordering:") + print( + "Order by registration_date DESC (regular), then profile_age ASC (nested), then profile_name ASC (nested):" + ) + complex_order = flat_df.set_index( + ["registration_date", "profile_age", "profile_name"], + ascending=[False, True, True], + ) + result = complex_order.to_pandas()[ + ["customer_id", "profile_name", "profile_age", "registration_date"] + ] + print(result) + print("\nThis shows:") + print("- Primary sort: newest registrations first (regular column)") + print("- Secondary sort: youngest first within same date (nested field)") + print("- Tertiary sort: alphabetical by name for same age (nested field)") + + +if __name__ == "__main__": + print("\n" + "LEANFRAME INDEXING EXAMPLES (NO PANDAS INGESTION)" + "\n") + print("This demo uses Arrow/Ibis for data creation and pandas only for output") + + backend = ibis.duckdb.connect() + session = leanframe.Session(backend=backend) + + try: + # Run all demos + demo_basic_indexing(session, backend) + demo_nested_data_with_indexing(session, backend) + demo_joins_with_indexing(session, backend) + demo_chaining_operations(session, backend) + demo_error_cases(session, backend) + demo_multi_column_ordering(session, backend) + + print("\n" + "=" * 70) + print("All demos completed!") + print("=" * 70) + print("\nKey Takeaways:") + print("1. Always set index explicitly for deterministic ordering") + print("2. Use .iloc for position-based access (with ordering)") + print("3. Use .loc for value-based filtering (on index column)") + print("4. Indexing works seamlessly with nested data extraction") + print("5. Chain operations: extract -> flatten -> index -> slice") + print("6. Multi-column ordering: like SQL ORDER BY col1 DESC, col2 ASC") + print("\nSee docs/indexing_guide.md for more details!") + finally: + backend.disconnect() diff --git a/docs/architecture/table_first_api_v2.md b/docs/architecture/table_first_api_v2.md new file mode 100644 index 0000000..9461271 --- /dev/null +++ b/docs/architecture/table_first_api_v2.md @@ -0,0 +1,212 @@ +# LeanFrame Table-First API Proposal v2 + +This document proposes a clearer split between `DataFrame`, `DataFrameHandler`, and `NestedHandler` without introducing a facade. The goal is to keep the current codebase intact while defining a cleaner target architecture that stays lazy, table-first, and Ibis-centered. + +## Core idea + +LeanFrame should think in terms of tables and expressions, not pandas objects. Users should be able to write familiar column- and table-oriented code, while Ibis remains the execution engine and source of truth for lazy evaluation. + +### Responsibilities + +#### `DataFrame` +The `DataFrame` object should stay small and expression-oriented. + +Keep here: +- `columns` +- `dtypes` +- `__getitem__` for column access +- `assign` +- `to_ibis` +- `to_pandas` +- `set_index` +- `iloc`, `loc` +- `head`, `tail` via `HeadTailMixin` + +Do not put here: +- nested schema traversal +- flattening logic +- cross-table orchestration +- backend lineage tracking +- record iteration helpers that force materialization beyond explicit conversion + +#### `DataFrameHandler` +This should be a nested-data capability object for one table. + +Keep here: +- schema introspection +- nested path discovery +- nested field name mapping +- flatten / extract planning +- nested-aware filtering that returns a new `DataFrame` +- backend qualifier metadata if you want that state attached to a single table + +Do not make it a second `DataFrame`. +It should not become the place for general table operations. + +#### `NestedHandler` +This should remain orchestration only. + +Keep here: +- named table registry +- prepare workflows +- multi-table joins +- lineage tracking +- backend reference coordination across named tables + +## Suggested module layout + +This keeps the current files, but adds focused v2 modules rather than growing one class. + +```text +leanframe/ + core/ + frame.py + indexing.py + nested_handler.py + schema_ops_v2.py + nested_ops_v2.py + table_ops_v2.py +``` + +### Module roles + +#### `schema_ops_v2.py` +Shared helpers for nested introspection. + +Examples: +- discover struct columns +- walk nested field paths +- map nested path -> extracted column name +- produce metadata without materializing data + +#### `nested_ops_v2.py` +Functional helpers for flattening and nested-aware selection. + +Examples: +- extract all nested fields +- extract selected nested fields +- filter by extracted columns +- return a new `DataFrame` only + +#### `table_ops_v2.py` +Optional future home for table-oriented transformations that are not nested-specific. + +Examples: +- projection helpers +- rename helpers +- lightweight column utilities +- future joins or aggregates if they should not live on `DataFrame` + +## Suggested API shape + +### DataFrame +Keep the public object compact and predictable. + +```python +class DataFrame(HeadTailMixin): + def __getitem__(self, key: str): ... + def assign(self, **kwargs): ... + def set_index(self, columns, ascending=True, name=None): ... + def to_ibis(self): ... + def to_pandas(self): ... +``` + +If you want one or two extra convenience methods, they should still stay in the “table-first” spirit: +- `select` +- `rename` +- `drop` + +But only if they map cleanly to Ibis and do not introduce pandas-style complexity. + +### DataFrameHandler +Keep it narrow and explicit. + +```python +class DataFrameHandler: + def show_structure(self): ... + def extract_nested_fields(self, verbose=True) -> DataFrame: ... + def filter_by(self, **kwargs) -> DataFrameHandler: ... + def get_extracted_column_name(self, nested_path: str) -> str | None: ... + def get_backend_info(self) -> dict[str, str | None]: ... +``` + +Potential refinement: +- `columns` should return metadata-derived fields only if that is cheap +- otherwise expose `extracted_fields` and leave `columns` to the extracted `DataFrame` + +### NestedHandler +Keep this as a coordinator. + +```python +class NestedHandler: + def add(self, name: str, df: DataFrame, ...): ... + def get(self, name: str) -> DataFrameHandler: ... + def prepare(self, name: str, fields=None, verbose=False) -> DataFrame: ... + def join(self, tables, on=None, predicates=None, how="inner") -> DataFrame: ... +``` + +## Naming rules + +Use names that reflect table work, not pandas imitation. + +Good: +- `prepare` +- `extract_nested_fields` +- `filter_by` +- `set_index` +- `select` +- `rename` +- `join` + +Avoid: +- multi-index semantics +- hierarchical pandas-like behavior +- methods whose main purpose is to mimic pandas internals + +## Growth model for new functionality + +Add functionality in layers, not by adding methods everywhere. + +### Layer 1: core table operations +These belong closest to `DataFrame`. +- projection +- filtering +- renaming +- ordering/indexing +- limiting +- joins when they are basic and Ibis-native + +### Layer 2: nested-specific operations +These belong to `DataFrameHandler` or nested helper modules. +- introspection +- flattening +- nested field selection +- nested-aware filter helpers + +### Layer 3: orchestration +These belong to `NestedHandler`. +- table registry +- joins across named tables +- prepare workflows +- lineage / backend tracking + +## What this means for the current code + +No facade is needed. +Instead, the implementation can remain discoverable by keeping the current classes public and moving the heavy logic into focused helper modules over time. + +The current file structure can be improved gradually: +- keep `frame.py` as the user-facing anchor for `DataFrame` +- keep `nested_handler.py` as the orchestration anchor +- extract reusable logic into `_v2` helper modules +- leave the old path in place until the split feels natural + +## Practical benefit + +This gives LeanFrame a simple story: +- `DataFrame` is the lazy table object +- `DataFrameHandler` understands nested schema shape +- `NestedHandler` coordinates multiple tables +- Ibis performs the actual relational work + +That keeps the mental model table-first, lazy, and explicit without drifting into pandas-style complexity. diff --git a/leanframe/core/frame.py b/leanframe/core/frame.py index 5447afc..238a133 100644 --- a/leanframe/core/frame.py +++ b/leanframe/core/frame.py @@ -616,11 +616,11 @@ def get_column(self, column_name: str) -> list: For efficiency, call extract_nested_fields() once and work with that. """ extracted = self._extract_nested_fields_silent() - pandas_df = extracted.to_pandas() - if column_name not in pandas_df.columns: - available = ", ".join(pandas_df.columns) + if column_name not in extracted._data.columns: + available = ", ".join(extracted._data.columns) raise KeyError(f"Column '{column_name}' not found. Available: {available}") - return pandas_df[column_name].tolist() + arrow_table = extracted._data.select(column_name).to_pyarrow() + return arrow_table[column_name].to_pylist() def get_record(self, index: int) -> dict: """ @@ -629,11 +629,17 @@ def get_record(self, index: int) -> dict: WARNING: Performs extraction and materialization on EVERY call. Not efficient for iterating over records - use extract_nested_fields() instead. """ + if index < 0: + raise IndexError("Negative indices are not supported") + extracted = self._extract_nested_fields_silent() - pandas_df = extracted.to_pandas() - if index >= len(pandas_df): - raise IndexError(f"Index {index} out of range (0-{len(pandas_df) - 1})") - return pandas_df.iloc[index].to_dict() + row_table = extracted._data.limit(1, offset=index).to_pyarrow() + if row_table.num_rows == 0: + total_rows = int(extracted._data.count().execute()) + raise IndexError(f"Index {index} out of range (0-{total_rows - 1})") + + row_data = row_table.to_pydict() + return {name: values[0] for name, values in row_data.items()} def __len__(self) -> int: """ @@ -642,8 +648,7 @@ def __len__(self) -> int: Note: Uses original DataFrame to avoid extraction overhead. """ # Use original DataFrame to avoid extraction overhead - pandas_df = self.original_df.to_pandas() - return len(pandas_df) + return int(self.original_df._data.count().execute()) def __getitem__(self, index: int) -> dict: """Get record by index."""