diff --git a/examples/cost-analysis-export/main.go b/examples/cost-analysis-export/main.go index 53876bddf..619b91b87 100644 --- a/examples/cost-analysis-export/main.go +++ b/examples/cost-analysis-export/main.go @@ -330,6 +330,7 @@ func (a *App) importAKSData(ctx context.Context) error { Prefix: &a.Config.AzureStorageAKSDataPrefix, }) + matched := 0 filesProcessed := 0 for pager.More() { page, err := pager.NextPage(ctx) @@ -348,6 +349,7 @@ func (a *App) importAKSData(ctx context.Context) error { continue } + matched++ if err := a.processAKSBlob(ctx, *blob.Name); err != nil { slog.Error("failed to process AKS blob", "name", *blob.Name, "error", err) continue @@ -357,11 +359,13 @@ func (a *App) importAKSData(ctx context.Context) error { } if filesProcessed == 0 { - slog.Error("no AKS export files found", "prefix", a.Config.AzureStorageAKSDataPrefix, "expected_pattern", a.Config.AzureStorageAKSDataPrefix+"export-*.csv") - } else { - slog.Info("processed AKS export files", "count", filesProcessed) + prefix := a.Config.AzureStorageAKSDataPrefix + if matched == 0 { + return fmt.Errorf("no AKS export files found under prefix %q (expected %sexport-*.csv or %sexport-*.csv.gz)", prefix, prefix, prefix) + } + return fmt.Errorf("no AKS export files imported under prefix %q (%d matching blob(s) failed)", prefix, matched) } - + slog.Info("processed AKS export files", "count", filesProcessed) return nil } @@ -375,6 +379,7 @@ func (a *App) importCostManagementData(ctx context.Context) error { Prefix: &a.Config.AzureStorageCostExportPrefix, }) + matched := 0 filesProcessed := 0 for pager.More() { @@ -394,6 +399,7 @@ func (a *App) importCostManagementData(ctx context.Context) error { continue } + matched++ if err := a.processCostManagementBlob(ctx, *blob.Name); err != nil { slog.Error("failed to process cost management blob", "name", *blob.Name, "error", err) continue @@ -403,11 +409,13 @@ func (a *App) importCostManagementData(ctx context.Context) error { } if filesProcessed == 0 { - slog.Error("no cost management files found to process", "prefix", a.Config.AzureStorageCostExportPrefix) - } else { - slog.Info("processed cost management files", "count", filesProcessed) + prefix := a.Config.AzureStorageCostExportPrefix + if matched == 0 { + return fmt.Errorf("no cost management files found under prefix %q (expected *.csv or *.csv.gz)", prefix) + } + return fmt.Errorf("no cost management files imported under prefix %q (%d matching blob(s) failed)", prefix, matched) } - + slog.Info("processed cost management files", "count", filesProcessed) return nil } @@ -511,7 +519,7 @@ func (a *App) ImportCSV(ctx context.Context, data io.Reader, tableName string) e standardHeader := []string{"SubscriptionGuid", "ResourceGroup", "ResourceLocation", "UsageDateTime", "MeterCategory", "MeterSubCategory", "MeterId", "MeterName", "MeterRegion", "UsageQuantity", "ResourceRate", "PreTaxCost", "ConsumedService", "ResourceType", "InstanceId", "Tags", "OfferId", "AdditionalInfo", "ServiceInfo1", "ServiceInfo2", "ServiceName", "ServiceTier", "Currency", "UnitOfMeasure"} return a.createTableFromHeader(ctx, tableName, standardHeader) } - return nil + return fmt.Errorf("empty CSV for table %s", tableName) } return fmt.Errorf("reading header: %w", err) } diff --git a/examples/cost-analysis-export/main_test.go b/examples/cost-analysis-export/main_test.go index a607e9b0f..83e69e022 100644 --- a/examples/cost-analysis-export/main_test.go +++ b/examples/cost-analysis-export/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "database/sql" "fmt" "io" "net/http" @@ -418,6 +419,99 @@ test-guid,rg,westus,2025-06-18,Virtual Machines,Standard,meter-id,VM,westus,1,10 } } +func TestImportCSV_EmptyAKSSplitsReturnsError(t *testing.T) { + app := &App{DB: openTestDB(t)} + err := app.ImportCSV(context.Background(), strings.NewReader(""), "aks_splits") + require.Error(t, err) + assert.Contains(t, err.Error(), "aks_splits") +} + +func TestImportCSV_EmptyCostManagementCreatesTable(t *testing.T) { + app := &App{DB: openTestDB(t)} + err := app.ImportCSV(context.Background(), strings.NewReader(""), "cost_management") + require.NoError(t, err) + cols, err := app.getTableColumns(context.Background(), "cost_management") + require.NoError(t, err) + assert.Contains(t, cols, "SubscriptionGuid") +} + +func TestApp_Merge_NoAKSExportFiles(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-aks", map[string][]byte{ + "cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"), + }), + AzureStorageContainerName: "test-no-aks", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no AKS export files found") + assert.Contains(t, err.Error(), ".csv.gz") +} + +func TestApp_Merge_EmptyAKSExport(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-empty-aks", map[string][]byte{ + "cost-analysis/export-2025-06-18.csv": []byte(""), + "cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"), + }), + AzureStorageContainerName: "test-empty-aks", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no AKS export files imported") + assert.Contains(t, err.Error(), "1 matching blob") +} + +func TestApp_Merge_NoCostManagementFiles(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-cost", map[string][]byte{ + "cost-analysis/export-2025-06-18.csv": []byte("Date,ID,Name,Kind,Fraction,SplitBucket,SplitKey\n2025-06-18,/subscriptions/test/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss,vmss,compute,1,usage,{}\n"), + }), + AzureStorageContainerName: "test-no-cost", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no cost management files found") + assert.Contains(t, err.Error(), ".csv.gz") +} + +func openTestDB(t *testing.T) *sql.DB { + t.Helper() + path := t.TempDir() + "/importcsv.db" + db, err := sql.Open("sqlite3", path) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + require.NoError(t, db.Ping()) + return db +} + func TestConfig_Validate(t *testing.T) { tests := []struct { name string