From a41d235792e5b560f63d60f7a3389ed3cdfb0a69 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Wed, 9 Sep 2026 09:48:51 -0700 Subject: [PATCH 1/2] cost-analysis-export: Fail merge when AKS or cost imports are empty Return an error from importAKSData and importCostManagementData when zero files are processed, and from ImportCSV when an AKS CSV is empty. Empty Cost Management files still create the standard header table. Introduced in #5393 (2025-10-24). Follow-up to #5771. Signed-off-by: Sebastien Tardif --- examples/cost-analysis-export/main.go | 14 ++-- examples/cost-analysis-export/main_test.go | 91 ++++++++++++++++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/examples/cost-analysis-export/main.go b/examples/cost-analysis-export/main.go index 53876bddf..251c2f6cd 100644 --- a/examples/cost-analysis-export/main.go +++ b/examples/cost-analysis-export/main.go @@ -357,11 +357,9 @@ func (a *App) importAKSData(ctx context.Context) error { } if filesProcessed == 0 { - slog.Error("no AKS export files found", "prefix", a.Config.AzureStorageAKSDataPrefix, "expected_pattern", a.Config.AzureStorageAKSDataPrefix+"export-*.csv") - } else { - slog.Info("processed AKS export files", "count", filesProcessed) + return fmt.Errorf("no AKS export files found under prefix %q (expected %sexport-*.csv)", a.Config.AzureStorageAKSDataPrefix, a.Config.AzureStorageAKSDataPrefix) } - + slog.Info("processed AKS export files", "count", filesProcessed) return nil } @@ -403,11 +401,9 @@ func (a *App) importCostManagementData(ctx context.Context) error { } if filesProcessed == 0 { - slog.Error("no cost management files found to process", "prefix", a.Config.AzureStorageCostExportPrefix) - } else { - slog.Info("processed cost management files", "count", filesProcessed) + return fmt.Errorf("no cost management files found under prefix %q", a.Config.AzureStorageCostExportPrefix) } - + slog.Info("processed cost management files", "count", filesProcessed) return nil } @@ -511,7 +507,7 @@ func (a *App) ImportCSV(ctx context.Context, data io.Reader, tableName string) e standardHeader := []string{"SubscriptionGuid", "ResourceGroup", "ResourceLocation", "UsageDateTime", "MeterCategory", "MeterSubCategory", "MeterId", "MeterName", "MeterRegion", "UsageQuantity", "ResourceRate", "PreTaxCost", "ConsumedService", "ResourceType", "InstanceId", "Tags", "OfferId", "AdditionalInfo", "ServiceInfo1", "ServiceInfo2", "ServiceName", "ServiceTier", "Currency", "UnitOfMeasure"} return a.createTableFromHeader(ctx, tableName, standardHeader) } - return nil + return fmt.Errorf("empty CSV for table %s", tableName) } return fmt.Errorf("reading header: %w", err) } diff --git a/examples/cost-analysis-export/main_test.go b/examples/cost-analysis-export/main_test.go index a607e9b0f..59c65823d 100644 --- a/examples/cost-analysis-export/main_test.go +++ b/examples/cost-analysis-export/main_test.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "database/sql" "fmt" "io" "net/http" @@ -418,6 +419,96 @@ test-guid,rg,westus,2025-06-18,Virtual Machines,Standard,meter-id,VM,westus,1,10 } } +func TestImportCSV_EmptyAKSSplitsReturnsError(t *testing.T) { + app := &App{DB: openTestDB(t)} + err := app.ImportCSV(context.Background(), strings.NewReader(""), "aks_splits") + require.Error(t, err) + assert.Contains(t, err.Error(), "aks_splits") +} + +func TestImportCSV_EmptyCostManagementCreatesTable(t *testing.T) { + app := &App{DB: openTestDB(t)} + err := app.ImportCSV(context.Background(), strings.NewReader(""), "cost_management") + require.NoError(t, err) + cols, err := app.getTableColumns(context.Background(), "cost_management") + require.NoError(t, err) + assert.Contains(t, cols, "SubscriptionGuid") +} + +func TestApp_Merge_NoAKSExportFiles(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-aks", map[string][]byte{ + "cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"), + }), + AzureStorageContainerName: "test-no-aks", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no AKS export files") +} + +func TestApp_Merge_EmptyAKSExport(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-empty-aks", map[string][]byte{ + "cost-analysis/export-2025-06-18.csv": []byte(""), + "cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"), + }), + AzureStorageContainerName: "test-empty-aks", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no AKS export files") +} + +func TestApp_Merge_NoCostManagementFiles(t *testing.T) { + date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC) + cfg := Config{ + AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-cost", map[string][]byte{ + "cost-analysis/export-2025-06-18.csv": []byte("Date,ID,Name,Kind,Fraction,SplitBucket,SplitKey\n2025-06-18,/subscriptions/test/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss,vmss,compute,1,usage,{}\n"), + }), + AzureStorageContainerName: "test-no-cost", + AzureStorageAKSDataPrefix: "cost-analysis/", + AzureStorageCostExportPrefix: "cost-management/", + AzureStorageResultFile: "cost-analysis/result.csv", + SQLiteFilePath: t.TempDir() + "/test.sqlite", + ExportDate: &date, + Timeout: time.Minute, + } + app, err := NewApp(cfg) + require.NoError(t, err) + err = app.Merge(context.Background()) + require.Error(t, err) + assert.Contains(t, err.Error(), "no cost management files") +} + +func openTestDB(t *testing.T) *sql.DB { + t.Helper() + path := t.TempDir() + "/importcsv.db" + db, err := sql.Open("sqlite3", path) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + require.NoError(t, db.Ping()) + return db +} + func TestConfig_Validate(t *testing.T) { tests := []struct { name string From 7628ca11720cfc627a4d7287d2e9a92469aca0d2 Mon Sep 17 00:00:00 2001 From: Sebastien Tardif Date: Wed, 9 Sep 2026 14:22:36 -0700 Subject: [PATCH 2/2] cost-analysis-export: Distinguish missing vs failed imports filesProcessed==0 used to say no files found even when matching blobs existed but ImportCSV failed (empty AKS CSV). Count matches separately and mention .csv.gz in the not-found path. Signed-off-by: Sebastien Tardif --- examples/cost-analysis-export/main.go | 16 ++++++++++++++-- examples/cost-analysis-export/main_test.go | 9 ++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/examples/cost-analysis-export/main.go b/examples/cost-analysis-export/main.go index 251c2f6cd..619b91b87 100644 --- a/examples/cost-analysis-export/main.go +++ b/examples/cost-analysis-export/main.go @@ -330,6 +330,7 @@ func (a *App) importAKSData(ctx context.Context) error { Prefix: &a.Config.AzureStorageAKSDataPrefix, }) + matched := 0 filesProcessed := 0 for pager.More() { page, err := pager.NextPage(ctx) @@ -348,6 +349,7 @@ func (a *App) importAKSData(ctx context.Context) error { continue } + matched++ if err := a.processAKSBlob(ctx, *blob.Name); err != nil { slog.Error("failed to process AKS blob", "name", *blob.Name, "error", err) continue @@ -357,7 +359,11 @@ func (a *App) importAKSData(ctx context.Context) error { } if filesProcessed == 0 { - return fmt.Errorf("no AKS export files found under prefix %q (expected %sexport-*.csv)", a.Config.AzureStorageAKSDataPrefix, a.Config.AzureStorageAKSDataPrefix) + prefix := a.Config.AzureStorageAKSDataPrefix + if matched == 0 { + return fmt.Errorf("no AKS export files found under prefix %q (expected %sexport-*.csv or %sexport-*.csv.gz)", prefix, prefix, prefix) + } + return fmt.Errorf("no AKS export files imported under prefix %q (%d matching blob(s) failed)", prefix, matched) } slog.Info("processed AKS export files", "count", filesProcessed) return nil @@ -373,6 +379,7 @@ func (a *App) importCostManagementData(ctx context.Context) error { Prefix: &a.Config.AzureStorageCostExportPrefix, }) + matched := 0 filesProcessed := 0 for pager.More() { @@ -392,6 +399,7 @@ func (a *App) importCostManagementData(ctx context.Context) error { continue } + matched++ if err := a.processCostManagementBlob(ctx, *blob.Name); err != nil { slog.Error("failed to process cost management blob", "name", *blob.Name, "error", err) continue @@ -401,7 +409,11 @@ func (a *App) importCostManagementData(ctx context.Context) error { } if filesProcessed == 0 { - return fmt.Errorf("no cost management files found under prefix %q", a.Config.AzureStorageCostExportPrefix) + prefix := a.Config.AzureStorageCostExportPrefix + if matched == 0 { + return fmt.Errorf("no cost management files found under prefix %q (expected *.csv or *.csv.gz)", prefix) + } + return fmt.Errorf("no cost management files imported under prefix %q (%d matching blob(s) failed)", prefix, matched) } slog.Info("processed cost management files", "count", filesProcessed) return nil diff --git a/examples/cost-analysis-export/main_test.go b/examples/cost-analysis-export/main_test.go index 59c65823d..83e69e022 100644 --- a/examples/cost-analysis-export/main_test.go +++ b/examples/cost-analysis-export/main_test.go @@ -453,7 +453,8 @@ func TestApp_Merge_NoAKSExportFiles(t *testing.T) { require.NoError(t, err) err = app.Merge(context.Background()) require.Error(t, err) - assert.Contains(t, err.Error(), "no AKS export files") + assert.Contains(t, err.Error(), "no AKS export files found") + assert.Contains(t, err.Error(), ".csv.gz") } func TestApp_Merge_EmptyAKSExport(t *testing.T) { @@ -475,7 +476,8 @@ func TestApp_Merge_EmptyAKSExport(t *testing.T) { require.NoError(t, err) err = app.Merge(context.Background()) require.Error(t, err) - assert.Contains(t, err.Error(), "no AKS export files") + assert.Contains(t, err.Error(), "no AKS export files imported") + assert.Contains(t, err.Error(), "1 matching blob") } func TestApp_Merge_NoCostManagementFiles(t *testing.T) { @@ -496,7 +498,8 @@ func TestApp_Merge_NoCostManagementFiles(t *testing.T) { require.NoError(t, err) err = app.Merge(context.Background()) require.Error(t, err) - assert.Contains(t, err.Error(), "no cost management files") + assert.Contains(t, err.Error(), "no cost management files found") + assert.Contains(t, err.Error(), ".csv.gz") } func openTestDB(t *testing.T) *sql.DB {