Skip to content

cost-analysis-export: Fail merge when AKS or cost imports are empty #5959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sebastien Tardif (SebTardif) wants to merge 2 commits into
base: master
Choose a base branch
from
+111 −9
Open

cost-analysis-export: Fail merge when AKS or cost imports are empty #5959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a41d235
cost-analysis-export: Fail merge when AKS or cost imports are empty
SebTardif Sep 9, 2026
7628ca1
cost-analysis-export: Distinguish missing vs failed imports
SebTardif Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions examples/cost-analysis-export/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,7 @@ func (a *App) importAKSData(ctx context.Context) error {
Prefix: &a.Config.AzureStorageAKSDataPrefix,
})

matched := 0
filesProcessed := 0
for pager.More() {
page, err := pager.NextPage(ctx)
Expand All @@ -348,6 +349,7 @@ func (a *App) importAKSData(ctx context.Context) error {
continue
}

matched++
if err := a.processAKSBlob(ctx, *blob.Name); err != nil {
slog.Error("failed to process AKS blob", "name", *blob.Name, "error", err)
continue
Expand All @@ -357,11 +359,13 @@ func (a *App) importAKSData(ctx context.Context) error {
}

if filesProcessed == 0 {
slog.Error("no AKS export files found", "prefix", a.Config.AzureStorageAKSDataPrefix, "expected_pattern", a.Config.AzureStorageAKSDataPrefix+"export-*.csv")
} else {
slog.Info("processed AKS export files", "count", filesProcessed)
prefix := a.Config.AzureStorageAKSDataPrefix
if matched == 0 {
return fmt.Errorf("no AKS export files found under prefix %q (expected %sexport-*.csv or %sexport-*.csv.gz)", prefix, prefix, prefix)
}
return fmt.Errorf("no AKS export files imported under prefix %q (%d matching blob(s) failed)", prefix, matched)
}

slog.Info("processed AKS export files", "count", filesProcessed)
return nil
}

Expand All @@ -375,6 +379,7 @@ func (a *App) importCostManagementData(ctx context.Context) error {
Prefix: &a.Config.AzureStorageCostExportPrefix,
})

matched := 0
filesProcessed := 0

for pager.More() {
Expand All @@ -394,6 +399,7 @@ func (a *App) importCostManagementData(ctx context.Context) error {
continue
}

matched++
if err := a.processCostManagementBlob(ctx, *blob.Name); err != nil {
slog.Error("failed to process cost management blob", "name", *blob.Name, "error", err)
continue
Expand All @@ -403,11 +409,13 @@ func (a *App) importCostManagementData(ctx context.Context) error {
}

if filesProcessed == 0 {
slog.Error("no cost management files found to process", "prefix", a.Config.AzureStorageCostExportPrefix)
} else {
slog.Info("processed cost management files", "count", filesProcessed)
prefix := a.Config.AzureStorageCostExportPrefix
if matched == 0 {
return fmt.Errorf("no cost management files found under prefix %q (expected *.csv or *.csv.gz)", prefix)
}
return fmt.Errorf("no cost management files imported under prefix %q (%d matching blob(s) failed)", prefix, matched)
}

slog.Info("processed cost management files", "count", filesProcessed)
return nil
}

Expand Down Expand Up @@ -511,7 +519,7 @@ func (a *App) ImportCSV(ctx context.Context, data io.Reader, tableName string) e
standardHeader := []string{"SubscriptionGuid", "ResourceGroup", "ResourceLocation", "UsageDateTime", "MeterCategory", "MeterSubCategory", "MeterId", "MeterName", "MeterRegion", "UsageQuantity", "ResourceRate", "PreTaxCost", "ConsumedService", "ResourceType", "InstanceId", "Tags", "OfferId", "AdditionalInfo", "ServiceInfo1", "ServiceInfo2", "ServiceName", "ServiceTier", "Currency", "UnitOfMeasure"}
return a.createTableFromHeader(ctx, tableName, standardHeader)
}
return nil
return fmt.Errorf("empty CSV for table %s", tableName)
}
return fmt.Errorf("reading header: %w", err)
}
Expand Down
94 changes: 94 additions & 0 deletions examples/cost-analysis-export/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"compress/gzip"
"context"
"database/sql"
"fmt"
"io"
"net/http"
Expand Down Expand Up @@ -418,6 +419,99 @@ test-guid,rg,westus,2025-06-18,Virtual Machines,Standard,meter-id,VM,westus,1,10
}
}

func TestImportCSV_EmptyAKSSplitsReturnsError(t *testing.T) {
app := &App{DB: openTestDB(t)}
err := app.ImportCSV(context.Background(), strings.NewReader(""), "aks_splits")
require.Error(t, err)
assert.Contains(t, err.Error(), "aks_splits")
}

func TestImportCSV_EmptyCostManagementCreatesTable(t *testing.T) {
app := &App{DB: openTestDB(t)}
err := app.ImportCSV(context.Background(), strings.NewReader(""), "cost_management")
require.NoError(t, err)
cols, err := app.getTableColumns(context.Background(), "cost_management")
require.NoError(t, err)
assert.Contains(t, cols, "SubscriptionGuid")
}

func TestApp_Merge_NoAKSExportFiles(t *testing.T) {
date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC)
cfg := Config{
AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-aks", map[string][]byte{
"cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"),
}),
AzureStorageContainerName: "test-no-aks",
AzureStorageAKSDataPrefix: "cost-analysis/",
AzureStorageCostExportPrefix: "cost-management/",
AzureStorageResultFile: "cost-analysis/result.csv",
SQLiteFilePath: t.TempDir() + "/test.sqlite",
ExportDate: &date,
Timeout: time.Minute,
}
app, err := NewApp(cfg)
require.NoError(t, err)
err = app.Merge(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "no AKS export files found")
assert.Contains(t, err.Error(), ".csv.gz")
}

func TestApp_Merge_EmptyAKSExport(t *testing.T) {
date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC)
cfg := Config{
AzureStorageConnectionString: SetupAzuriteContainer(t, "test-empty-aks", map[string][]byte{
"cost-analysis/export-2025-06-18.csv": []byte(""),
"cost-management/file1.csv": []byte("SubscriptionGuid,ResourceGroup,ResourceLocation,UsageDateTime,MeterCategory,MeterSubCategory,MeterId,MeterName,MeterRegion,UsageQuantity,ResourceRate,PreTaxCost,ConsumedService,ResourceType,InstanceId,Tags,OfferId,AdditionalInfo,ServiceInfo1,ServiceInfo2,ServiceName,ServiceTier,Currency,UnitOfMeasure\n"),
}),
AzureStorageContainerName: "test-empty-aks",
AzureStorageAKSDataPrefix: "cost-analysis/",
AzureStorageCostExportPrefix: "cost-management/",
AzureStorageResultFile: "cost-analysis/result.csv",
SQLiteFilePath: t.TempDir() + "/test.sqlite",
ExportDate: &date,
Timeout: time.Minute,
}
app, err := NewApp(cfg)
require.NoError(t, err)
err = app.Merge(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "no AKS export files imported")
assert.Contains(t, err.Error(), "1 matching blob")
}

func TestApp_Merge_NoCostManagementFiles(t *testing.T) {
date := time.Date(2025, 6, 18, 0, 0, 0, 0, time.UTC)
cfg := Config{
AzureStorageConnectionString: SetupAzuriteContainer(t, "test-no-cost", map[string][]byte{
"cost-analysis/export-2025-06-18.csv": []byte("Date,ID,Name,Kind,Fraction,SplitBucket,SplitKey\n2025-06-18,/subscriptions/test/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/vmss,vmss,compute,1,usage,{}\n"),
}),
AzureStorageContainerName: "test-no-cost",
AzureStorageAKSDataPrefix: "cost-analysis/",
AzureStorageCostExportPrefix: "cost-management/",
AzureStorageResultFile: "cost-analysis/result.csv",
SQLiteFilePath: t.TempDir() + "/test.sqlite",
ExportDate: &date,
Timeout: time.Minute,
}
app, err := NewApp(cfg)
require.NoError(t, err)
err = app.Merge(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "no cost management files found")
assert.Contains(t, err.Error(), ".csv.gz")
}

func openTestDB(t *testing.T) *sql.DB {
t.Helper()
path := t.TempDir() + "/importcsv.db"
db, err := sql.Open("sqlite3", path)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
require.NoError(t, db.Ping())
return db
}

func TestConfig_Validate(t *testing.T) {
tests := []struct {
name string
Expand Down
Loading