setup_environment.sh

#!/bin/bash

echo "========================================="
echo "PySpark & GraphFrames Demo Setup"
echo "========================================="
echo ""

# Check Python version
echo "Checking Python version..."
PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}')
echo "Found Python $PYTHON_VERSION"

if ! command -v python3 &> /dev/null; then
    echo "Error: Python 3 is not installed. Please install Python 3.8 or higher."
    exit 1
fi

# Create virtual environment
echo ""
echo "Creating virtual environment..."
python3 -m venv venv

# Activate virtual environment
echo "Activating virtual environment..."
source venv/bin/activate

# Upgrade pip
echo ""
echo "Upgrading pip..."
pip install --upgrade pip

# Install requirements
echo ""
echo "Installing Python dependencies..."
pip install -r requirements.txt

# Set environment variables
echo ""
echo "Setting up environment variables..."
export PYSPARK_PYTHON=$(which python3)
export PYSPARK_DRIVER_PYTHON=$(which python3)

# Add to activation script for future use
cat >> venv/bin/activate << 'EOF'

# PySpark environment variables
export PYSPARK_PYTHON="$VIRTUAL_ENV/bin/python"
export PYSPARK_DRIVER_PYTHON="$VIRTUAL_ENV/bin/python"
export SPARK_LOCAL_IP="127.0.0.1"
EOF

echo ""
echo "========================================="
echo "Setup Complete!"
echo "========================================="
echo ""
echo "To activate the environment, run:"
echo "  source venv/bin/activate"
echo ""
echo "To generate synthetic data, run:"
echo "  python -c 'from src.data_generator import generate_all_datasets; generate_all_datasets()'"
echo ""
echo "To start Jupyter notebook, run:"
echo "  jupyter notebook notebooks/comprehensive_demo.ipynb"
echo ""