test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jcli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 
73     int      buildStatus;
74     int      runStatus;
75     string   buildOut;
76     string   runOut;
77 }
78 
79 /++ CONFIGURATION ++/
80 auto TEST_CASES = 
81 [
82     testCase().inFolder         ("./00-basic-usage-default-command/")
83               .withParams       ("20")
84               .expectStatusToBe (0)
85               .finish           (),
86     testCase().inFolder         ("./00-basic-usage-default-command/")
87               .withParams       ("20 --reverse")
88               .expectStatusToBe (128)
89               .finish           (),
90     
91     testCase().inFolder         ("./01-named-sub-commands/")
92               .withParams       ("return 0")
93               .expectStatusToBe (0)
94               .finish           (),
95     testCase().inFolder         ("./01-named-sub-commands/")
96               .withParams       ("r 128")
97               .expectStatusToBe (128)
98               .finish           (),
99 
100     testCase().inFolder         ("./02-shorthand-longhand-args/")
101               .withParams       ("return --code 0")
102               .expectStatusToBe (0)
103               .finish           (),
104     testCase().inFolder         ("./02-shorthand-longhand-args/")
105               .withParams       ("r -c=128")
106               .expectStatusToBe (128)
107               .finish           (),
108 
109     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2")
112               .expectStatusToBe (3)
113               .allowToFail      ()
114               .finish           (),
115     testCase().inFolder         ("./03-inheritence-base-commands/")
116               .withParams       ("add 1 2 --offset=7")
117               .expectStatusToBe (10)
118               .allowToFail      ()
119               .finish           (),
120 
121     testCase().inFolder         ("./04-custom-arg-binders/")
122               .withParams       ("./dub.sdl")
123               .expectStatusToBe (0)
124               .finish           (),
125     testCase().inFolder         ("./04-custom-arg-binders/")
126               .withParams       ("./lalaland.txt")
127               .expectStatusToBe (-1)
128               .finish           (),
129 
130     testCase().inFolder         ("./05-built-in-binders/")
131               .withParams       ("echo -b -i 2 -f 2.2 -s Hola -e red")
132               .expectStatusToBe (0)
133               .finish           (),
134 
135     testCase().inFolder         ("./06-result-with-fail-code/")
136               .withParams       ("abc")
137             //   .expectStatusToBe (100)
138               .expectStatusToBe (-1)
139               .finish           (),
140     testCase().inFolder         ("./06-result-with-fail-code/")
141               .withParams       ("0")
142             //   .expectStatusToBe (102)
143               .expectStatusToBe (-1)
144               .finish           (),
145     testCase().inFolder         ("./06-result-with-fail-code/")
146               .withParams       ("3")
147             //   .expectStatusToBe (101)
148               .expectStatusToBe (-1)
149               .finish           (),
150     testCase().inFolder         ("./06-result-with-fail-code/")
151               .withParams       ("1")
152               .expectStatusToBe (0)
153               .finish           (),
154 
155     testCase().inFolder             ("./08-arg-binder-validation")
156               .withParams           ("20 69")
157               .expectStatusToBe     (0)
158               .finish               (),
159     testCase().inFolder             ("./08-arg-binder-validation")
160               .withParams           ("69 69")
161               .expectStatusToBe     (-1)
162               .expectOutputToMatch  ("Expected number to be even")
163               .finish               (),
164     testCase().inFolder             ("./08-arg-binder-validation")
165               .withParams           ("20 20")
166               .expectStatusToBe     (-1)
167               .expectOutputToMatch  ("Expected number to be odd")
168               .finish               (),
169 
170     testCase().inFolder             ("./09-raw-unparsed-arg-list")
171               .withParams           ("echo -- Some args")
172               .expectStatusToBe     (0)
173               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
174               .finish               (),
175     testCase().inFolder             ("./09-raw-unparsed-arg-list")
176               .withParams           ("noarg")
177               .expectStatusToBe     (0)
178               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
179               .finish               (),
180 
181     testCase().inFolder             ("./10-argument-options")
182               .withParams           ("sensitive --abc 2")
183               .expectStatusToBe     (0)
184               .finish               (),
185     testCase().inFolder             ("./10-argument-options")
186               .withParams           ("sensitive --abC 2")
187               .expectStatusToBe     (-1)
188               .finish               (),
189     testCase().inFolder             ("./10-argument-options")
190               .withParams           ("insensitive --abc 2")
191               .expectStatusToBe     (0)
192               .finish               (),
193     testCase().inFolder             ("./10-argument-options")
194               .withParams           ("insensitive --ABC 2")
195               .expectStatusToBe     (0)
196               .finish               (),
197     testCase().inFolder             ("./10-argument-options")
198               .withParams           ("redefine --abc 2")
199               .expectStatusToBe     (2)
200               .finish               (),
201     testCase().inFolder             ("./10-argument-options")
202               .withParams           ("redefine --abc 2 --abc 1")
203               .expectStatusToBe     (1)
204               .finish               (),
205     testCase().inFolder             ("./10-argument-options")
206               .withParams           ("no-redefine --abc 2")
207               .expectStatusToBe     (0)
208               .finish               (),
209     testCase().inFolder             ("./10-argument-options")
210               .withParams           ("no-redefine --abc 2 --abc 1")
211               .expectStatusToBe     (-1)
212               .finish               (),
213 ];
214  
215 /++ MAIN ++/
216 int main(string[] args)
217 {
218     return (new CommandLineInterface!test()).parseAndExecute(args);
219 }
220 
221 /++ COMMANDS ++/
222 @CommandDefault("Runs all test cases")
223 struct DefaultCommand
224 {
225     int onExecute()
226     {
227         writefln("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
228         const anyfails = runTestSet(TEST_CASES);
229 
230         return anyfails ? -1 : 0;
231     }
232 }
233 
234 @Command("ui", "Runs the test UI")
235 struct UICommand
236 {
237     void onExecute()
238     {
239         auto ui = new UI();
240         ui.run();
241     }
242 }
243 
244 @Command("cleanup", "Runs the cleanup command for all test cases")
245 struct CleanupCommand
246 {
247     void onExecute()
248     {
249         foreach(test; TEST_CASES)
250             runCleanup(test);
251     }
252 }
253 
254 /++ FUNCS ++/
255 bool runTestSet(TestCase[] testSet)
256 {
257     auto results = new TestResult[testSet.length];
258     foreach(i, testCase; testSet)
259         results[i] = testCase.runTest();
260 
261     if(results.any!(r => !r.passed))
262         writefln("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
263 
264     size_t failedCount;
265     foreach(failed; results.filter!(r => !r.passed))
266     {
267         failedCount++;
268         writefln("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
269 
270         foreach(reason; failed.failedReasons)
271             writefln("\t\t- %s".ansi.fg(Ansi4BitColour.red).to!string, reason);
272     }
273 
274     size_t passedCount = results.length - failedCount;
275     writefln(
276         "\n%s %s, %s %s, %s total tests",
277         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
278         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
279         results.length
280     );
281 
282     return (failedCount != 0);
283 }
284 
285 TestResult runTest(TestCase testCase)
286 {
287     const    results = getBuildAndTestResults(testCase);
288     auto     passed  = true;
289     string[] reasons;
290 
291     void failIf(bool condition, string reason)
292     {
293         if(!condition)
294             return;
295 
296         passed   = false;
297         reasons ~= reason;
298     }
299 
300     failIf(results[0].status != 0, "Build failed.");
301 
302     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
303     // a -1 as expecting -1 or 255.
304     if(testCase.expectedStatus.get(0) != -1)
305         failIf(results[1].status != testCase.expectedStatus.get(0), "Status code is wrong.");
306     else
307         failIf(results[1].status != -1 && results[1].status != 255, "Status code is wrong. (-1 special case)");
308 
309     if(!testCase.outputRegex.isNull)
310         failIf(!results[1].output.matchFirst(testCase.outputRegex.get), "Output doesn't contain a match for the given regex.");
311 
312     if(testCase.allowedToFail)
313     {
314         if(!passed)
315             writeln("Test FAILED (ALLOWED).".ansi.fg(Ansi4BitColour.yellow));
316         passed = true;
317     }
318 
319     if(!passed)
320         writeln("Test FAILED".ansi.fg(Ansi4BitColour.red));
321     else
322         writefln("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
323 
324     return TestResult(passed, reasons, testCase, results[0].status, results[1].status, results[0].output, results[1].output);
325 }
326 
327 // [0] = build result, [1] = test result
328 auto getBuildAndTestResults(TestCase testCase)
329 {
330     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
331     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
332     const RESULT_COLOUR   = Ansi4BitColour.yellow;
333 
334     writefln("");
335     writefln("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
336     writefln("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
337     writefln("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
338     writefln("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
339     writefln("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
340 
341     auto cwd = getcwd();
342     chdir(testCase.folder);
343     scope(exit) chdir(cwd);
344 
345     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
346     {
347         writefln("Cleanup: %s".ansi.fg(Ansi4BitColour.brightBlack).to!string, file);
348         remove(file);
349     }
350 
351     const buildString   = "dub build --compiler=ldc2";
352     const commandString = "\"./test\" " ~ testCase.params;
353     const buildResult   = executeShell(buildString);
354     const result        = executeShell(commandString);
355 
356     writefln("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.status.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
357     writefln("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.status.to!string.ansi.fg(RESULT_COLOUR),      result.output);
358 
359     return [buildResult, result];
360 }
361 
362 void runCleanup(TestCase testCase)
363 {
364     auto cwd = getcwd();
365     chdir(testCase.folder);
366     scope(exit) chdir(cwd);
367     executeShell("dub clean");
368 }
369 
370 final class UI
371 {
372     TextBuffer buffer;
373     Layout layout;
374     size_t selectedTest;
375     size_t buildOffset;
376     size_t buildXOffset;
377     size_t runOffset;
378 
379     static struct Test
380     {
381         enum State
382         {
383             notRan,
384             success,
385             failure
386         }
387         string displayName;
388         State state;
389         int buildStatus;
390         int runStatus;
391         string buildOutput;
392         string runOutput;
393     }
394     Test[] tests;
395 
396     void run()
397     {
398         Console.attach();
399         this.buffer = Console.createTextBuffer();
400         this.layout = Layout(
401             Rect(0, 0, this.buffer.width, this.buffer.height),
402             8, 8
403         );
404 
405         this.tests = TEST_CASES.map!(t => Test(" " ~ t.folder ~ " " ~ t.params ~ " ")).array;
406 
407         while(Console.isAttached)
408         {
409             Console.processEvents((e)
410             {
411                 e.match!(
412                     (ConsoleKeyEvent key) => handleKey(key),
413                     (_){}
414                 );
415             });
416 
417             if(Console.isAttached)
418                 this.draw();
419         }
420     }
421 
422     private void handleKey(ConsoleKeyEvent key)
423     {
424         if(!key.isDown)
425             return;
426 
427         // If I weren't lazy I'd bring the shift and ctrl keys into play, but I'm lazy.
428         if(key.key == ConsoleKey.escape)
429             Console.detach();
430         else if(key.key == ConsoleKey.up && this.selectedTest != 0)
431             this.selectedTest--;
432         else if(key.key == ConsoleKey.down && this.selectedTest < TEST_CASES.length-1)
433             this.selectedTest++;
434         else if(key.key == ConsoleKey.enter)
435             this.runTest(this.selectedTest);
436         else if(key.key == ConsoleKey.home && this.buildOffset != 0)
437             this.buildOffset--;
438         else if(key.key == ConsoleKey.end)
439             this.buildOffset++;
440         else if(key.key == ConsoleKey.insert && this.runOffset != 0)
441             this.runOffset--;
442         else if(key.key == ConsoleKey.del)
443             this.runOffset++;
444         else if(key.key == ConsoleKey.right)
445             this.buildXOffset++;
446         else if(key.key == ConsoleKey.left && this.buildXOffset != 0)
447             this.buildXOffset--;
448         else if(key.key == ConsoleKey.back)
449         {
450             foreach(i; 0..TEST_CASES.length)
451                 this.runTest(i);
452         }
453     }
454 
455     private void runTest(size_t test)
456     {
457         const TestResult result = .runTest(TEST_CASES[test]);
458         this.tests[test].state = (result.passed) ? Test.State.success : Test.State.failure;
459         this.tests[test].buildOutput = result.buildOut;
460         this.tests[test].runOutput = result.runOut;
461     }
462 
463     private void draw()
464     {
465         auto testBlock = BorderWidgetBuilder()
466             .withBackground(AnsiColour(Ansi4BitColour.black))
467             .withForeground(AnsiColour(Ansi4BitColour.yellow))
468             .withBlockArea(Rect(0, 0, 3, 7))
469             .withBorderStyle(BorderStyle.all)
470             .withTitle("TESTS")
471             .withTitleAlignment(Alignment.center)
472             .build();
473 
474         auto resultBlock = BorderWidgetBuilder()
475             .withBackground(AnsiColour(Ansi4BitColour.black))
476             .withForeground(AnsiColour(Ansi4BitColour.yellow))
477             .withBlockArea(Rect(3, 0, 8, 7))
478             .withBorderStyle(BorderStyle.all)
479             .withTitle("RESULT")
480             .withTitleAlignment(Alignment.center)
481             .build();
482 
483         testBlock.render(this.layout, this.buffer);
484         resultBlock.render(this.layout, this.buffer);
485 
486         const testArea = testBlock.innerArea(this.layout);
487         const testLayout = Layout(testArea, testArea.width, testArea.height);
488         foreach(i, test; this.tests[this.selectedTest..$])
489         {
490             const fg = 
491                 (test.state == Test.State.notRan)
492                     ? AnsiColour.init
493                     : (test.state == Test.State.failure)
494                         ? AnsiColour(Ansi4BitColour.red)
495                         : AnsiColour(Ansi4BitColour.green);
496 
497             TextWidgetBuilder()
498                 .withBlockArea(Rect(2, cast(int)i, testArea.width, cast(int)i+1))
499                 .withText(test.displayName)
500                 .withStyle(i == 0 ? AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)).fg(fg) : AnsiStyleSet.init.fg(fg))
501                 .build()
502                 .render(testLayout, this.buffer);
503         }
504 
505         const resultArea   = resultBlock.innerArea(this.layout);
506         const resultLayout = Layout(resultArea, 2, 1);
507         const buildArea    = resultLayout.blockRectToRealRect(Rect(0, 0, 1, 1));
508         const runArea      = resultLayout.blockRectToRealRect(Rect(1, 0, 2, 1));
509         const buildLayout  = Layout(buildArea, 1, 1);
510         const runLayout    = Layout(runArea, 1, 1);
511 
512         auto buildBlock = BorderWidgetBuilder()
513             .withBlockArea(Rect(0, 0, 1, 1))
514             .withBorderStyle(BorderStyle.all)
515             .withTitle("Build")
516             .withTitleAlignment(Alignment.center)
517             .build();
518         buildBlock.render(buildLayout, this.buffer);
519 
520         auto runBlock = BorderWidgetBuilder()
521             .withBlockArea(Rect(0, 0, 1, 1))
522             .withBorderStyle(BorderStyle.all)
523             .withTitle("Run")
524             .withTitleAlignment(Alignment.center)
525             .build();
526         runBlock.render(runLayout, this.buffer);
527 
528         const buildOutArea      = buildBlock.innerArea(buildLayout);
529         const runOutArea        = runBlock.innerArea(runLayout);
530         const buildOutLayout    = Layout(buildOutArea, 1, buildOutArea.height);
531         const runOutLayout      = Layout(runOutArea, 1, runOutArea.height);
532 
533         foreach(i, line; this.tests[this.selectedTest].buildOutput.lineSplitter.drop(this.buildOffset).enumerate)
534         {
535             TextWidgetBuilder()
536                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
537                 .withText(line[min(this.buildXOffset, line.length)..$])
538                 .build()
539                 .render(buildOutLayout, this.buffer);
540         }
541 
542         foreach(i, line; this.tests[this.selectedTest].runOutput.lineSplitter.drop(this.runOffset).enumerate)
543         {
544             TextWidgetBuilder()
545                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
546                 .withText(line[min(this.buildXOffset, line.length)..$])
547                 .build()
548                 .render(runOutLayout, this.buffer);
549         }
550 
551         ShortcutsWidgetBuilder!7()
552             .withBackground(AnsiColour(Ansi4BitColour.brightBlack))
553             .withKeyStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)))
554             .withDescriptionStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.brightBlack)))
555             .withShortcut(0, "ESC", "Close UI")
556             .withShortcut(1, "↑↓", "Select test")
557             .withShortcut(2, "ENTER", "Run test")
558             .withShortcut(3, "BACKSPACE", "Run all tests")
559             .withShortcut(4, "INS DEL", "Move Build Output")
560             .withShortcut(5, "PGUP PGDN", "Move Run Output")
561             .withShortcut(6, "←→", "Offset Output")
562             .build()
563             .render(this.buffer);
564 
565         this.buffer.refresh();
566     }
567 }