test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jcli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 
73     int      buildStatus;
74     int      runStatus;
75     string   buildOut;
76     string   runOut;
77 }
78 
79 /++ CONFIGURATION ++/
80 auto TEST_CASES = 
81 [
82     testCase().inFolder         ("./00-basic-usage-default-command/")
83               .withParams       ("20")
84               .expectStatusToBe (0)
85               .finish           (),
86     testCase().inFolder         ("./00-basic-usage-default-command/")
87               .withParams       ("20 --reverse")
88               .expectStatusToBe (128)
89               .finish           (),
90     
91     testCase().inFolder         ("./01-named-sub-commands/")
92               .withParams       ("return 0")
93               .expectStatusToBe (0)
94               .finish           (),
95     testCase().inFolder         ("./01-named-sub-commands/")
96               .withParams       ("r 128")
97               .expectStatusToBe (128)
98               .finish           (),
99 
100     testCase().inFolder         ("./02-shorthand-longhand-args/")
101               .withParams       ("return --code 0")
102               .expectStatusToBe (0)
103               .finish           (),
104     testCase().inFolder         ("./02-shorthand-longhand-args/")
105               .withParams       ("r -c=128")
106               .expectStatusToBe (128)
107               .finish           (),
108 
109     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2")
112               .expectStatusToBe (3)
113               .allowToFail      ()
114               .finish           (),
115     testCase().inFolder         ("./03-inheritence-base-commands/")
116               .withParams       ("add 1 2 --offset=7")
117               .expectStatusToBe (10)
118               .allowToFail      ()
119               .finish           (),
120 
121     testCase().inFolder         ("./04-custom-arg-binders/")
122               .withParams       ("./dub.sdl")
123               .expectStatusToBe (0)
124               .finish           (),
125     testCase().inFolder         ("./04-custom-arg-binders/")
126               .withParams       ("./lalaland.txt")
127               .expectStatusToBe (-1)
128               .finish           (),
129 
130     testCase().inFolder         ("./05-built-in-binders/")
131               .withParams       ("echo -b -i 2 -f 2.2 -s Hola -e red")
132               .expectStatusToBe (0)
133               .finish           (),
134 
135     testCase().inFolder         ("./06-result-with-fail-code/")
136               .withParams       ("abc")
137               .expectStatusToBe (100)
138               .finish           (),
139     testCase().inFolder         ("./06-result-with-fail-code/")
140               .withParams       ("0")
141               .expectStatusToBe (102)
142               .finish           (),
143     testCase().inFolder         ("./06-result-with-fail-code/")
144               .withParams       ("3")
145               .expectStatusToBe (101)
146               .finish           (),
147     testCase().inFolder         ("./06-result-with-fail-code/")
148               .withParams       ("1")
149               .expectStatusToBe (0)
150               .finish           (),
151 
152     testCase().inFolder             ("./08-arg-binder-validation")
153               .withParams           ("20 69")
154               .expectStatusToBe     (0)
155               .finish               (),
156     testCase().inFolder             ("./08-arg-binder-validation")
157               .withParams           ("69 69")
158               .expectStatusToBe     (-1)
159               .expectOutputToMatch  ("Expected number to be even")
160               .finish               (),
161     testCase().inFolder             ("./08-arg-binder-validation")
162               .withParams           ("20 20")
163               .expectStatusToBe     (-1)
164               .expectOutputToMatch  ("Expected number to be odd")
165               .finish               (),
166 
167     testCase().inFolder             ("./09-raw-unparsed-arg-list")
168               .withParams           ("echo -- Some args")
169               .expectStatusToBe     (0)
170               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
171               .finish               (),
172     testCase().inFolder             ("./09-raw-unparsed-arg-list")
173               .withParams           ("noarg")
174               .expectStatusToBe     (0)
175               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
176               .finish               (),
177 
178     testCase().inFolder             ("./10-argument-options")
179               .withParams           ("sensitive --abc 2")
180               .expectStatusToBe     (0)
181               .finish               (),
182     testCase().inFolder             ("./10-argument-options")
183               .withParams           ("sensitive --abC 2")
184               .expectStatusToBe     (-1)
185               .finish               (),
186     testCase().inFolder             ("./10-argument-options")
187               .withParams           ("insensitive --abc 2")
188               .expectStatusToBe     (0)
189               .finish               (),
190     testCase().inFolder             ("./10-argument-options")
191               .withParams           ("insensitive --ABC 2")
192               .expectStatusToBe     (0)
193               .finish               (),
194     testCase().inFolder             ("./10-argument-options")
195               .withParams           ("redefine --abc 2")
196               .expectStatusToBe     (2)
197               .finish               (),
198     testCase().inFolder             ("./10-argument-options")
199               .withParams           ("redefine --abc 2 --abc 1")
200               .expectStatusToBe     (1)
201               .finish               (),
202     testCase().inFolder             ("./10-argument-options")
203               .withParams           ("no-redefine --abc 2")
204               .expectStatusToBe     (0)
205               .finish               (),
206     testCase().inFolder             ("./10-argument-options")
207               .withParams           ("no-redefine --abc 2 --abc 1")
208               .expectStatusToBe     (-1)
209               .finish               (),
210 ];
211  
212 /++ MAIN ++/
213 int main(string[] args)
214 {
215     return (new CommandLineInterface!test()).parseAndExecute(args);
216 }
217 
218 /++ COMMANDS ++/
219 @CommandDefault("Runs all test cases")
220 struct DefaultCommand
221 {
222     int onExecute()
223     {
224         writefln("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
225         const anyfails = runTestSet(TEST_CASES);
226 
227         return anyfails ? -1 : 0;
228     }
229 }
230 
231 @Command("ui", "Runs the test UI")
232 struct UICommand
233 {
234     void onExecute()
235     {
236         auto ui = new UI();
237         ui.run();
238     }
239 }
240 
241 @Command("cleanup", "Runs the cleanup command for all test cases")
242 struct CleanupCommand
243 {
244     void onExecute()
245     {
246         foreach(test; TEST_CASES)
247             runCleanup(test);
248     }
249 }
250 
251 /++ FUNCS ++/
252 bool runTestSet(TestCase[] testSet)
253 {
254     auto results = new TestResult[testSet.length];
255     foreach(i, testCase; testSet)
256         results[i] = testCase.runTest();
257 
258     if(results.any!(r => !r.passed))
259         writefln("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
260 
261     size_t failedCount;
262     foreach(failed; results.filter!(r => !r.passed))
263     {
264         failedCount++;
265         writefln("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
266 
267         foreach(reason; failed.failedReasons)
268             writefln("\t\t- %s".ansi.fg(Ansi4BitColour.red).to!string, reason);
269     }
270 
271     size_t passedCount = results.length - failedCount;
272     writefln(
273         "\n%s %s, %s %s, %s total tests",
274         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
275         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
276         results.length
277     );
278 
279     return (failedCount != 0);
280 }
281 
282 TestResult runTest(TestCase testCase)
283 {
284     const    results = getBuildAndTestResults(testCase);
285     auto     passed  = true;
286     string[] reasons;
287 
288     void failIf(bool condition, string reason)
289     {
290         if(!condition)
291             return;
292 
293         passed   = false;
294         reasons ~= reason;
295     }
296 
297     failIf(results[0].status != 0, "Build failed.");
298 
299     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
300     // a -1 as expecting -1 or 255.
301     if(testCase.expectedStatus.get(0) != -1)
302         failIf(results[1].status != testCase.expectedStatus.get(0), "Status code is wrong.");
303     else
304         failIf(results[1].status != -1 && results[1].status != 255, "Status code is wrong. (-1 special case)");
305 
306     if(!testCase.outputRegex.isNull)
307         failIf(!results[1].output.matchFirst(testCase.outputRegex.get), "Output doesn't contain a match for the given regex.");
308 
309     if(testCase.allowedToFail)
310     {
311         if(!passed)
312             writeln("Test FAILED (ALLOWED).".ansi.fg(Ansi4BitColour.yellow));
313         passed = true;
314     }
315 
316     if(!passed)
317         writeln("Test FAILED".ansi.fg(Ansi4BitColour.red));
318     else
319         writefln("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
320 
321     return TestResult(passed, reasons, testCase, results[0].status, results[1].status, results[0].output, results[1].output);
322 }
323 
324 // [0] = build result, [1] = test result
325 auto getBuildAndTestResults(TestCase testCase)
326 {
327     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
328     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
329     const RESULT_COLOUR   = Ansi4BitColour.yellow;
330 
331     writefln("");
332     writefln("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
333     writefln("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
334     writefln("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
335     writefln("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
336     writefln("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
337 
338     auto cwd = getcwd();
339     chdir(testCase.folder);
340     scope(exit) chdir(cwd);
341 
342     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
343     {
344         writefln("Cleanup: %s".ansi.fg(Ansi4BitColour.brightBlack).to!string, file);
345         remove(file);
346     }
347 
348     const buildString   = "dub build --compiler=ldc2";
349     const commandString = "\"./test\" " ~ testCase.params;
350     const buildResult   = executeShell(buildString);
351     const result        = executeShell(commandString);
352 
353     writefln("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.status.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
354     writefln("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.status.to!string.ansi.fg(RESULT_COLOUR),      result.output);
355 
356     return [buildResult, result];
357 }
358 
359 void runCleanup(TestCase testCase)
360 {
361     auto cwd = getcwd();
362     chdir(testCase.folder);
363     scope(exit) chdir(cwd);
364     executeShell("dub clean");
365 }
366 
367 final class UI
368 {
369     TextBuffer buffer;
370     Layout layout;
371     size_t selectedTest;
372     size_t buildOffset;
373     size_t buildXOffset;
374     size_t runOffset;
375 
376     static struct Test
377     {
378         enum State
379         {
380             notRan,
381             success,
382             failure
383         }
384         string displayName;
385         State state;
386         int buildStatus;
387         int runStatus;
388         string buildOutput;
389         string runOutput;
390     }
391     Test[] tests;
392 
393     void run()
394     {
395         Console.attach();
396         this.buffer = Console.createTextBuffer();
397         this.layout = Layout(
398             Rect(0, 0, this.buffer.width, this.buffer.height),
399             8, 8
400         );
401 
402         this.tests = TEST_CASES.map!(t => Test(" " ~ t.folder ~ " " ~ t.params ~ " ")).array;
403 
404         while(Console.isAttached)
405         {
406             Console.processEvents((e)
407             {
408                 e.match!(
409                     (ConsoleKeyEvent key) => handleKey(key),
410                     (_){}
411                 );
412             });
413 
414             if(Console.isAttached)
415                 this.draw();
416         }
417     }
418 
419     private void handleKey(ConsoleKeyEvent key)
420     {
421         if(!key.isDown)
422             return;
423 
424         // If I weren't lazy I'd bring the shift and ctrl keys into play, but I'm lazy.
425         if(key.key == ConsoleKey.escape)
426             Console.detach();
427         else if(key.key == ConsoleKey.up && this.selectedTest != 0)
428             this.selectedTest--;
429         else if(key.key == ConsoleKey.down && this.selectedTest < TEST_CASES.length-1)
430             this.selectedTest++;
431         else if(key.key == ConsoleKey.enter)
432             this.runTest(this.selectedTest);
433         else if(key.key == ConsoleKey.home && this.buildOffset != 0)
434             this.buildOffset--;
435         else if(key.key == ConsoleKey.end)
436             this.buildOffset++;
437         else if(key.key == ConsoleKey.insert && this.runOffset != 0)
438             this.runOffset--;
439         else if(key.key == ConsoleKey.del)
440             this.runOffset++;
441         else if(key.key == ConsoleKey.right)
442             this.buildXOffset++;
443         else if(key.key == ConsoleKey.left && this.buildXOffset != 0)
444             this.buildXOffset--;
445         else if(key.key == ConsoleKey.back)
446         {
447             foreach(i; 0..TEST_CASES.length)
448                 this.runTest(i);
449         }
450     }
451 
452     private void runTest(size_t test)
453     {
454         const TestResult result = .runTest(TEST_CASES[test]);
455         this.tests[test].state = (result.passed) ? Test.State.success : Test.State.failure;
456         this.tests[test].buildOutput = result.buildOut;
457         this.tests[test].runOutput = result.runOut;
458     }
459 
460     private void draw()
461     {
462         auto testBlock = BorderWidgetBuilder()
463             .withBackground(AnsiColour(Ansi4BitColour.black))
464             .withForeground(AnsiColour(Ansi4BitColour.yellow))
465             .withBlockArea(Rect(0, 0, 3, 7))
466             .withBorderStyle(BorderStyle.all)
467             .withTitle("TESTS")
468             .withTitleAlignment(Alignment.center)
469             .build();
470 
471         auto resultBlock = BorderWidgetBuilder()
472             .withBackground(AnsiColour(Ansi4BitColour.black))
473             .withForeground(AnsiColour(Ansi4BitColour.yellow))
474             .withBlockArea(Rect(3, 0, 8, 7))
475             .withBorderStyle(BorderStyle.all)
476             .withTitle("RESULT")
477             .withTitleAlignment(Alignment.center)
478             .build();
479 
480         testBlock.render(this.layout, this.buffer);
481         resultBlock.render(this.layout, this.buffer);
482 
483         const testArea = testBlock.innerArea(this.layout);
484         const testLayout = Layout(testArea, testArea.width, testArea.height);
485         foreach(i, test; this.tests[this.selectedTest..$])
486         {
487             const fg = 
488                 (test.state == Test.State.notRan)
489                     ? AnsiColour.init
490                     : (test.state == Test.State.failure)
491                         ? AnsiColour(Ansi4BitColour.red)
492                         : AnsiColour(Ansi4BitColour.green);
493 
494             TextWidgetBuilder()
495                 .withBlockArea(Rect(2, cast(int)i, testArea.width, cast(int)i+1))
496                 .withText(test.displayName)
497                 .withStyle(i == 0 ? AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)).fg(fg) : AnsiStyleSet.init.fg(fg))
498                 .build()
499                 .render(testLayout, this.buffer);
500         }
501 
502         const resultArea   = resultBlock.innerArea(this.layout);
503         const resultLayout = Layout(resultArea, 2, 1);
504         const buildArea    = resultLayout.blockRectToRealRect(Rect(0, 0, 1, 1));
505         const runArea      = resultLayout.blockRectToRealRect(Rect(1, 0, 2, 1));
506         const buildLayout  = Layout(buildArea, 1, 1);
507         const runLayout    = Layout(runArea, 1, 1);
508 
509         auto buildBlock = BorderWidgetBuilder()
510             .withBlockArea(Rect(0, 0, 1, 1))
511             .withBorderStyle(BorderStyle.all)
512             .withTitle("Build")
513             .withTitleAlignment(Alignment.center)
514             .build();
515         buildBlock.render(buildLayout, this.buffer);
516 
517         auto runBlock = BorderWidgetBuilder()
518             .withBlockArea(Rect(0, 0, 1, 1))
519             .withBorderStyle(BorderStyle.all)
520             .withTitle("Run")
521             .withTitleAlignment(Alignment.center)
522             .build();
523         runBlock.render(runLayout, this.buffer);
524 
525         const buildOutArea      = buildBlock.innerArea(buildLayout);
526         const runOutArea        = runBlock.innerArea(runLayout);
527         const buildOutLayout    = Layout(buildOutArea, 1, buildOutArea.height);
528         const runOutLayout      = Layout(runOutArea, 1, runOutArea.height);
529 
530         foreach(i, line; this.tests[this.selectedTest].buildOutput.lineSplitter.drop(this.buildOffset).enumerate)
531         {
532             TextWidgetBuilder()
533                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
534                 .withText(line[min(this.buildXOffset, line.length)..$])
535                 .build()
536                 .render(buildOutLayout, this.buffer);
537         }
538 
539         foreach(i, line; this.tests[this.selectedTest].runOutput.lineSplitter.drop(this.runOffset).enumerate)
540         {
541             TextWidgetBuilder()
542                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
543                 .withText(line[min(this.buildXOffset, line.length)..$])
544                 .build()
545                 .render(runOutLayout, this.buffer);
546         }
547 
548         ShortcutsWidgetBuilder!7()
549             .withBackground(AnsiColour(Ansi4BitColour.brightBlack))
550             .withKeyStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)))
551             .withDescriptionStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.brightBlack)))
552             .withShortcut(0, "ESC", "Close UI")
553             .withShortcut(1, "↑↓", "Select test")
554             .withShortcut(2, "ENTER", "Run test")
555             .withShortcut(3, "BACKSPACE", "Run all tests")
556             .withShortcut(4, "INS DEL", "Move Build Output")
557             .withShortcut(5, "PGUP PGDN", "Move Run Output")
558             .withShortcut(6, "←→", "Offset Output")
559             .build()
560             .render(this.buffer);
561 
562         this.buffer.refresh();
563     }
564 }