| | | 1 | | using System.ComponentModel; |
| | | 2 | | using System.Diagnostics; |
| | | 3 | | |
| | | 4 | | using Spectre.Console.Cli; |
| | | 5 | | |
| | | 6 | | namespace NGql.Tool; |
| | | 7 | | |
| | | 8 | | /// <summary> |
| | | 9 | | /// Internal: runs every Fixtures/*.snippet, compares against its .expected, prints PASS/FAIL. |
| | | 10 | | /// Used by `make skill-eval` for regression testing inside the NGql repo. Hidden from --help. |
| | | 11 | | /// </summary> |
| | | 12 | | internal sealed class FixturesCommand : AsyncCommand<FixturesCommand.Settings> |
| | | 13 | | { |
| | | 14 | | public sealed class Settings : CommandSettings { } |
| | | 15 | | |
| | | 16 | | public override async Task<int> ExecuteAsync(CommandContext context, Settings settings) |
| | | 17 | | { |
| | | 18 | | var fixturesDir = FixtureSupport.ResolveFixturesDir(); |
| | | 19 | | if (!Directory.Exists(fixturesDir)) |
| | | 20 | | { |
| | | 21 | | await Console.Error.WriteLineAsync($"fixtures dir not found: {fixturesDir}"); |
| | | 22 | | return ExitCodes.FileNotFound; |
| | | 23 | | } |
| | | 24 | | |
| | | 25 | | var snippets = Directory.GetFiles(fixturesDir, "*.snippet").OrderBy(p => p).ToList(); |
| | | 26 | | if (snippets.Count == 0) |
| | | 27 | | { |
| | | 28 | | Console.WriteLine($"(no fixtures found under {fixturesDir})"); |
| | | 29 | | return ExitCodes.Ok; |
| | | 30 | | } |
| | | 31 | | |
| | | 32 | | var sw = Stopwatch.StartNew(); |
| | | 33 | | var pass = 0; |
| | | 34 | | var fail = 0; |
| | | 35 | | foreach (var snippetPath in snippets) |
| | | 36 | | { |
| | | 37 | | var name = Path.GetFileNameWithoutExtension(snippetPath); |
| | | 38 | | var expectedPath = Path.Combine(fixturesDir, $"{name}.expected"); |
| | | 39 | | if (await FixtureSupport.RunFixture(snippetPath, expectedPath)) |
| | | 40 | | pass++; |
| | | 41 | | else |
| | | 42 | | fail++; |
| | | 43 | | } |
| | | 44 | | |
| | | 45 | | sw.Stop(); |
| | | 46 | | Console.WriteLine(); |
| | | 47 | | Console.WriteLine($"{pass} passed, {fail} failed ({sw.ElapsedMilliseconds} ms)"); |
| | | 48 | | return fail == 0 ? ExitCodes.Ok : ExitCodes.RenderFailed; |
| | | 49 | | } |
| | | 50 | | } |
| | | 51 | | |
| | | 52 | | /// <summary> |
| | | 53 | | /// Internal: runs a single named fixture by basename (without extension). |
| | | 54 | | /// Used by contributors to debug a specific fixture. Hidden from --help. |
| | | 55 | | /// </summary> |
| | | 56 | | internal sealed class FixtureCommand : AsyncCommand<FixtureCommand.Settings> |
| | | 57 | | { |
| | | 58 | | public sealed class Settings : CommandSettings |
| | | 59 | | { |
| | | 60 | | [CommandArgument(0, "<NAME>")] |
| | | 61 | | [Description("Fixture basename (without extension).")] |
| | | 62 | | public string Name { get; init; } = string.Empty; |
| | | 63 | | } |
| | | 64 | | |
| | | 65 | | public override async Task<int> ExecuteAsync(CommandContext context, Settings settings) |
| | | 66 | | { |
| | | 67 | | var fixturesDir = FixtureSupport.ResolveFixturesDir(); |
| | | 68 | | var snippetPath = Path.Combine(fixturesDir, $"{settings.Name}.snippet"); |
| | | 69 | | var expectedPath = Path.Combine(fixturesDir, $"{settings.Name}.expected"); |
| | | 70 | | if (!File.Exists(snippetPath)) |
| | | 71 | | { |
| | | 72 | | await Console.Error.WriteLineAsync($"fixture not found: {snippetPath}"); |
| | | 73 | | return ExitCodes.FileNotFound; |
| | | 74 | | } |
| | | 75 | | |
| | | 76 | | return await FixtureSupport.RunFixture(snippetPath, expectedPath) |
| | | 77 | | ? ExitCodes.Ok |
| | | 78 | | : ExitCodes.RenderFailed; |
| | | 79 | | } |
| | | 80 | | } |
| | | 81 | | |
| | | 82 | | internal static class FixtureSupport |
| | | 83 | | { |
| | | 84 | | public static async Task<bool> RunFixture(string snippetPath, string expectedPath) |
| | | 85 | | { |
| | 147 | 86 | | var name = Path.GetFileNameWithoutExtension(snippetPath); |
| | 147 | 87 | | var snippet = await File.ReadAllTextAsync(snippetPath); |
| | 147 | 88 | | var result = await SnippetRunner.CompileAndRun(snippet); |
| | | 89 | | |
| | 147 | 90 | | if (!result.Ok) |
| | | 91 | | { |
| | 3 | 92 | | Console.WriteLine($"FAIL {name} (compile/run error)"); |
| | 18 | 93 | | foreach (var line in result.Error.Split('\n')) |
| | 6 | 94 | | Console.WriteLine($" {line}"); |
| | 3 | 95 | | return false; |
| | | 96 | | } |
| | | 97 | | |
| | 144 | 98 | | if (!File.Exists(expectedPath)) |
| | | 99 | | { |
| | 6 | 100 | | Console.WriteLine($"SMOKE {name} (no expected file; output below)"); |
| | 72 | 101 | | foreach (var line in result.Output.Split('\n')) |
| | 30 | 102 | | Console.WriteLine($" {line}"); |
| | 6 | 103 | | return true; |
| | | 104 | | } |
| | | 105 | | |
| | 138 | 106 | | var expected = (await File.ReadAllTextAsync(expectedPath)).Replace("\r\n", "\n").TrimEnd(); |
| | 138 | 107 | | var actual = result.Output.Replace("\r\n", "\n").TrimEnd(); |
| | | 108 | | |
| | 138 | 109 | | if (expected == actual) |
| | | 110 | | { |
| | 135 | 111 | | Console.WriteLine($"PASS {name}"); |
| | 135 | 112 | | return true; |
| | | 113 | | } |
| | | 114 | | |
| | 3 | 115 | | Console.WriteLine($"FAIL {name} (output mismatch)"); |
| | 3 | 116 | | Console.WriteLine($" --- expected ---"); |
| | 18 | 117 | | foreach (var line in expected.Split('\n')) |
| | 6 | 118 | | Console.WriteLine($" {line}"); |
| | 3 | 119 | | Console.WriteLine($" --- actual ---"); |
| | 36 | 120 | | foreach (var line in actual.Split('\n')) |
| | 15 | 121 | | Console.WriteLine($" {line}"); |
| | 3 | 122 | | return false; |
| | 147 | 123 | | } |
| | | 124 | | |
| | 75 | 125 | | public static string ResolveFixturesDir() => ResolveFixturesDir(ToolEnvironment.BaseDirectory()); |
| | | 126 | | |
| | | 127 | | public static string ResolveFixturesDir(string baseDirectory) |
| | | 128 | | { |
| | | 129 | | // When running via `dotnet run`, BaseDirectory is bin/Debug/...; fall back to a |
| | | 130 | | // project-relative Fixtures dir found by walking up to Tool.csproj. |
| | 84 | 131 | | var local = Path.Combine(baseDirectory, "Fixtures"); |
| | 84 | 132 | | if (Directory.Exists(local)) |
| | 75 | 133 | | return local; |
| | | 134 | | |
| | 9 | 135 | | var dir = baseDirectory; |
| | 42 | 136 | | while (dir is not null && !File.Exists(Path.Combine(dir, "Tool.csproj"))) |
| | 33 | 137 | | dir = Path.GetDirectoryName(dir); |
| | | 138 | | |
| | 9 | 139 | | return dir is null ? local : Path.Combine(dir, "Fixtures"); |
| | | 140 | | } |
| | | 141 | | } |