diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs index 36a1d17c..ec4abf3d 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/DataModelConverterService.cs @@ -18,7 +18,7 @@ namespace TALXIS.CLI.Features.Data.DataModelConverter; public class DataModelConverterService { private static readonly ILogger _logger = TxcLoggerFactory.CreateLogger(nameof(DataModelConverterService)); - private static readonly string[] SupportedFormats = ["dbml", "sql", "edmx", "ribbon"]; + private static readonly string[] SupportedFormats = ["dbml", "sql", "plainsql", "edmx", "ribbon"]; /// /// Parses a Power Platform solution from a solution project folder, a declarations @@ -259,7 +259,11 @@ public static ParsedModel ParseModelFolder(string folderPath) Module module = new(); // Get files named Entity.xml in subfolders - var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories); + // Ordered: Directory.GetFiles gives no ordering guarantee, so without this the + // table, relationship and enum order in the output varies by filesystem and the + // result cannot be committed or diffed. + var entityFiles = Directory.GetFiles(folderPath, "Entity.xml", SearchOption.AllDirectories) + .OrderBy(f => f, StringComparer.Ordinal).ToArray(); foreach (var file in entityFiles) { @@ -284,7 +288,7 @@ public static ParsedModel ParseModelFolder(string folderPath) // Get files in folder Other/Relationships (directory may not exist in scaffolded solutions) var relationshipsDir = Path.Combine(folderPath, "Other", "Relationships"); var relationshipFiles = Directory.Exists(relationshipsDir) - ? Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories) + ? [.. Directory.GetFiles(relationshipsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)] : Array.Empty(); foreach (var file in relationshipFiles) { @@ -302,7 +306,7 @@ public static ParsedModel ParseModelFolder(string folderPath) // Get files in folder called OptionSets (directory may not exist in scaffolded solutions) var optionsetsDir = Path.Combine(folderPath, "OptionSets"); var optionsetFiles = Directory.Exists(optionsetsDir) - ? Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories) + ? [.. Directory.GetFiles(optionsetsDir, "*.xml", SearchOption.AllDirectories).OrderBy(f => f, StringComparer.Ordinal)] : Array.Empty(); foreach (var file in optionsetFiles) { @@ -362,15 +366,20 @@ public static ParsedModel ParseModules(List modules) List EntityTables = ParseEntities(modules); List EntityOptionSets = ParseOptionSets(modules); - // Remove optionset rows without optionsets defined + // Downgrade optionset rows whose optionset is not resolvable here (declared with no + // options, owned by another module, or platform-owned) rather than dropping the column. var validOptionSetNames = EntityOptionSets.Select(x => x.LocalizedName).ToHashSet(StringComparer.OrdinalIgnoreCase); - foreach (var entity in EntityTables) + foreach (var row in EntityTables + .SelectMany(entity => entity.Rows) + .Where(row => + row.RowType is (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit) + && !validOptionSetNames.Contains(row.OptionSetName))) { - entity.Rows = [.. entity.Rows - .Where(row => - row.RowType is not (RowType.Picklist or RowType.Multiselectoptionset or RowType.State or RowType.Status or RowType.Bit) - || validOptionSetNames.Contains(row.OptionSetName) - )]; + // Clearing OptionSetName is enough and is all that is needed: it is what + // ToDbDiagramNotation prefers over RowType, so leaving it set would make the + // column reference an Enum that was never emitted. RowType is deliberately + // left alone so each translator keeps its own handling for the kind. + row.OptionSetName = string.Empty; } // Fill in setnames where missing with placeholder logical names @@ -420,6 +429,21 @@ public static List ParseRelationships(List modules, List ParseRelationships(List modules, List x.RowType == RowType.Primarykey), connectionTable, - connectionTable.Rows.FirstOrDefault(x => x.Name == firstEntityTable.LogicalName + "id")); + connectionTable.Rows.FirstOrDefault(x => x.Name == firstRowName)); - var secondToMid = new Relationship(relationship.Attribute("Name").Value, + var secondToMid = new Relationship(secondRelationshipName, "ManyToOne", secondEntityTable, secondEntityTable.Rows.FirstOrDefault(x => x.RowType == RowType.Primarykey), connectionTable, - connectionTable.Rows.FirstOrDefault(x => x.Name == secondEntityTable.LogicalName + "id")); + connectionTable.Rows.FirstOrDefault(x => x.Name == secondRowName)); EntityRelationships.Add(firstToMid); EntityRelationships.Add(secondToMid); @@ -489,7 +524,7 @@ public static List ParseRelationships(List modules, List x.RowType == RowType.Primarykey)); - if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.RighSideTable == entityRelationship.RighSideTable) == default) + if (EntityRelationships.FirstOrDefault(x => x.LeftSideTable == entityRelationship.LeftSideTable && x.LeftSideRow == entityRelationship.LeftSideRow && x.RighSideTable == entityRelationship.RighSideTable) == default) { EntityRelationships.Add(entityRelationship); } diff --git a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs index 4a8ea5d7..f8073c7f 100644 --- a/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs +++ b/src/TALXIS.CLI.Features.Data/DataModelConverter/Model/Module.cs @@ -11,8 +11,7 @@ public class Module { public Module() { - var random = new Random(); - Colorhex = string.Format("#{0:X6}", random.Next(0x1000000)); + Colorhex = ColourFor(ModuleName); } public Module(string module, XDocument xml) @@ -20,8 +19,7 @@ public Module(string module, XDocument xml) ModuleName = module; XmlDoc = xml; - var random = new Random(); - Colorhex = string.Format("#{0:X6}", random.Next(0x1000000)); + Colorhex = ColourFor(ModuleName); entities = XmlDoc.Descendants().Where(x => x.Name == "Entity").ToList(); relationships = XmlDoc.Descendants().Where(x => x.Name == "EntityRelationship").ToList(); @@ -36,4 +34,21 @@ public Module(string module, XDocument xml) public List optionsets = []; public string Colorhex { get; } + + /// + /// Derives the module colour from its name so the same input always converts to the + /// same bytes. A random colour made every conversion a spurious diff, which meant a + /// generated diagram could not be committed or compared across a model change. + /// FNV-1a rather than string.GetHashCode, which is randomised per process on .NET Core. + /// + private static string ColourFor(string moduleName) + { + uint hash = 2166136261; + foreach (var c in moduleName ?? string.Empty) + { + hash ^= c; + hash *= 16777619; + } + return string.Format("#{0:X6}", hash & 0xFFFFFF); + } } diff --git a/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs new file mode 100644 index 00000000..c764cc66 --- /dev/null +++ b/tests/TALXIS.CLI.Tests/Data/DataModelConverter/DataModelConverterServiceTests.cs @@ -0,0 +1,239 @@ +using System.IO; +using System.Linq; +using System.Reflection; +using DotMake.CommandLine; +using TALXIS.CLI.Features.Data; +using System.Xml.Linq; +using TALXIS.CLI.Features.Data.DataModelConverter; +using Model = TALXIS.CLI.Features.Data.DataModelConverter.Model; +using Xunit; + +namespace TALXIS.CLI.Tests.Data.DataModelConverter; + +/// +/// Regression tests for four defects that made the converter silently lose or corrupt +/// model content. Each was found by diffing converter output against the source +/// declarations of real solutions; each test fails against the unfixed converter. +/// +public class DataModelConverterServiceTests +{ + private static XElement Entity(string logicalName, params string[] attributes) + { + var attrXml = string.Join("", attributes); + return XElement.Parse($""" + + {logicalName} + + + + primarykey + {attrXml} + + + + + """); + } + + private static string Attr(string name, string type) => + $"""{type}"""; + + private static string Lookup(string name) => Attr(name, "lookup"); + + private static XElement OneToMany(string name, string child, string childAttr, string parent) => + XElement.Parse($""" + + OneToMany + {child} + {parent} + {childAttr} + + """); + + private static Model.Module ModuleWith(XElement[] entities, XElement[]? relationships = null) + { + var module = new Model.Module(); + module.entities.AddRange(entities); + if (relationships != null) module.relationships.AddRange(relationships); + return module; + } + + // ---- Defect 1: relationships deduped on the table pair, not the column ------------- + + [Fact] + public void TwoLookupsBetweenSameTablePair_BothProduceRelationships() + { + var module = ModuleWith( + [Entity("account"), Entity("contoso_project", Lookup("contoso_ownerid"), Lookup("contoso_billtoid"))], + [ + OneToMany("rel_owner", "contoso_project", "contoso_ownerid", "account"), + OneToMany("rel_billto", "contoso_project", "contoso_billtoid", "account"), + ]); + + var model = DataModelConverterService.ParseModules([module]); + + var toAccount = model.relationships + .Where(r => r.RighSideTable?.LogicalName == "account") + .Select(r => r.LeftSideRow?.Name) + .ToList(); + + Assert.Equal(2, toAccount.Count); + Assert.Contains("contoso_ownerid", toAccount); + Assert.Contains("contoso_billtoid", toAccount); + } + + // ---- Defect 3: a column vanishes when its option set will not resolve -------------- + + [Fact] + public void PicklistWithUnresolvableOptionSet_KeepsColumnInsteadOfDroppingIt() + { + var picklist = """ + + picklist + contoso_never_declared_anywhere + + """; + var module = ModuleWith([Entity("contoso_thing", picklist)]); + + var table = DataModelConverterService.ParseModules([module]) + .tables.Single(t => t.LogicalName == "contoso_thing"); + + var row = table.Rows.SingleOrDefault(r => r.Name == "contoso_statuscode"); + Assert.NotNull(row); + // Cleared so the column cannot reference an Enum that was never emitted; + // RowType is deliberately left alone so each translator keeps its own handling. + Assert.True(string.IsNullOrEmpty(row!.OptionSetName)); + } + + // ---- Defect 4: output was not reproducible — colours came from new Random() -------- + + [Fact] + public void ModuleColour_IsDerivedFromName_SoConversionIsReproducible() + { + var a = new Model.Module("Areas/Service/Project/Model", new XDocument(new XElement("root"))); + var b = new Model.Module("Areas/Service/Project/Model", new XDocument(new XElement("root"))); + var other = new Model.Module("Areas/Environment/Start/Model", new XDocument(new XElement("root"))); + + Assert.Equal(a.Colorhex, b.Colorhex); + Assert.NotEqual(a.Colorhex, other.Colorhex); + Assert.Matches("^#[0-9A-F]{6}$", a.Colorhex); + } + + // ---- Defect 5: a self-referencing N:N emitted duplicate columns and refs ----------- + + [Fact] + public void SelfReferencingManyToMany_ProducesTwoDistinctIntersectColumns() + { + var manyToMany = XElement.Parse(""" + + ManyToMany + contoso_thing + contoso_thing + contoso_thing_thing + + """); + var module = ModuleWith([Entity("contoso_thing")], [manyToMany]); + + var model = DataModelConverterService.ParseModules([module]); + var intersect = model.tables.Single(t => t.LogicalName == "contoso_thing_thing"); + + var names = intersect.Rows.Select(r => r.Name).ToList(); + Assert.Equal(names.Count, names.Distinct().Count()); + + // Two legs, each anchored on its own column — one shared column produced a + // duplicate endpoint pair, which a DBML parser rejects outright. + var legs = model.relationships.Where(r => r.RighSideTable?.LogicalName == "contoso_thing_thing").ToList(); + Assert.Equal(2, legs.Count); + Assert.Equal(2, legs.Select(l => l.RighSideRow?.Name).Distinct().Count()); + + // The legs also need distinct relationship names: EDMX renders the intersect side + // as NavigationProperty Name="{relationship.Name}", with a matching Partner and + // NavigationPropertyBinding Path, so sharing one name emits each of them twice. + Assert.Equal(2, legs.Select(l => l.Name).Distinct().Count()); + } + + [Fact] + public void SelfReferencingManyToMany_RendersDistinctNavigationPropertiesOnTheIntersect() + { + var manyToMany = XElement.Parse(""" + + ManyToMany + contoso_thing + contoso_thing + contoso_thing_thing + + """); + var module = ModuleWith([Entity("contoso_thing")], [manyToMany]); + + var model = DataModelConverterService.ParseModules([module]); + var edmx = DataModelConverterService.ConvertToEDMX(model); + + // The intersect's own EntityType carries one navigation property per leg. + var intersect = System.Text.RegularExpressions.Regex.Match( + edmx, "", + System.Text.RegularExpressions.RegexOptions.Singleline).Value; + var navNames = System.Text.RegularExpressions.Regex + .Matches(intersect, " m.Groups[1].Value).ToList(); + + Assert.Equal(2, navNames.Count); + Assert.Equal(2, navNames.Distinct().Count()); + } + + // ---- Every relationship endpoint must be non-null: the translators dereference ----- + // ---- LeftSideRow/RighSideRow without a null check. ------------------------------- + + [Fact] + public void EveryEmittedRelationship_HasBothEndpointsResolved() + { + var module = ModuleWith( + [Entity("account"), Entity("contoso_project", Lookup("contoso_ownerid"))], + [OneToMany("rel_owner", "contoso_project", "contoso_ownerid", "account")]); + + var model = DataModelConverterService.ParseModules([module]); + + Assert.All(model.relationships, r => + { + Assert.NotNull(r.LeftSideTable); + Assert.NotNull(r.LeftSideRow); + Assert.NotNull(r.RighSideTable); + Assert.NotNull(r.RighSideRow); + }); + } + + // ---- Every format the CLI advertises must actually convert ------------------------ + // plainsql was listed in the option's AllowedValues and fully implemented in the + // format switch, but missing from the service's SupportedFormats guard -- so it was + // rejected on every invocation, and the error message listed that same guard as truth. + + [Fact] + public void EveryAdvertisedTargetFormat_IsAcceptedByTheService() + { + var advertised = typeof(DataModelConvertCliCommand) + .GetProperty(nameof(DataModelConvertCliCommand.TargetFormat))! + .GetCustomAttribute()! + .AllowedValues! + .Cast() + .ToList(); + + Assert.NotEmpty(advertised); + + var dir = Path.Combine(Path.GetTempPath(), "txc-fmt-" + Path.GetRandomFileName()); + var entityDir = Path.Combine(dir, "Entities", "contoso_thing"); + Directory.CreateDirectory(entityDir); + File.WriteAllText(Path.Combine(entityDir, "Entity.xml"), Entity("contoso_thing").ToString()); + try + { + foreach (var format in advertised) + { + var outFile = Path.Combine(dir, "out." + format); + var ex = Record.Exception(() => DataModelConverterService.ConvertModel(dir, format, outFile)); + Assert.True(ex is null, $"--target {format} is advertised but failed: {ex?.Message}"); + } + } + finally + { + Directory.Delete(dir, recursive: true); + } + } +}