test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jcli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 
73     int      buildStatus;
74     int      runStatus;
75     string   buildOut;
76     string   runOut;
77 }
78 
79 /++ CONFIGURATION ++/
80 auto TEST_CASES = 
81 [
82     testCase().inFolder         ("./00-basic-usage-default-command/")
83               .withParams       ("20")
84               .expectStatusToBe (0)
85               .finish           (),
86     testCase().inFolder         ("./00-basic-usage-default-command/")
87               .withParams       ("20 --reverse")
88               .expectStatusToBe (128)
89               .finish           (),
90     
91     testCase().inFolder         ("./01-named-sub-commands/")
92               .withParams       ("return 0")
93               .expectStatusToBe (0)
94               .finish           (),
95     testCase().inFolder         ("./01-named-sub-commands/")
96               .withParams       ("r 128")
97               .expectStatusToBe (128)
98               .finish           (),
99 
100     testCase().inFolder         ("./02-shorthand-longhand-args/")
101               .withParams       ("return --code 0")
102               .expectStatusToBe (0)
103               .finish           (),
104     testCase().inFolder         ("./02-shorthand-longhand-args/")
105               .withParams       ("r -c=128")
106               .expectStatusToBe (128)
107               .finish           (),
108 
109     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2")
112               .expectStatusToBe (3)
113               .allowToFail      ()
114               .finish           (),
115     testCase().inFolder         ("./03-inheritence-base-commands/")
116               .withParams       ("add 1 2 --offset=7")
117               .expectStatusToBe (10)
118               .allowToFail      ()
119               .finish           (),
120 
121     testCase().inFolder         ("./04-custom-arg-binders/")
122               .withParams       ("./dub.sdl")
123               .expectStatusToBe (0)
124               .finish           (),
125     testCase().inFolder         ("./04-custom-arg-binders/")
126               .withParams       ("./lalaland.txt")
127               .expectStatusToBe (-1)
128               .finish           (),
129 
130     testCase().inFolder         ("./05-built-in-binders/")
131               .withParams       ("echo -b -i 2 -f 2.2 -s Hola -e red")
132               .expectStatusToBe (0)
133               .finish           (),
134 
135     testCase().inFolder             ("./08-arg-binder-validation")
136               .withParams           ("20 69")
137               .expectStatusToBe     (0)
138               .finish               (),
139     testCase().inFolder             ("./08-arg-binder-validation")
140               .withParams           ("69 69")
141               .expectStatusToBe     (-1)
142               .expectOutputToMatch  ("Expected number to be even")
143               .finish               (),
144     testCase().inFolder             ("./08-arg-binder-validation")
145               .withParams           ("20 20")
146               .expectStatusToBe     (-1)
147               .expectOutputToMatch  ("Expected number to be odd")
148               .finish               (),
149 
150     testCase().inFolder             ("./09-raw-unparsed-arg-list")
151               .withParams           ("echo -- Some args")
152               .expectStatusToBe     (0)
153               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
154               .finish               (),
155     testCase().inFolder             ("./09-raw-unparsed-arg-list")
156               .withParams           ("noarg")
157               .expectStatusToBe     (0)
158               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
159               .finish               (),
160 
161     testCase().inFolder             ("./10-argument-options")
162               .withParams           ("sensitive --abc 2")
163               .expectStatusToBe     (0)
164               .finish               (),
165     testCase().inFolder             ("./10-argument-options")
166               .withParams           ("sensitive --abC 2")
167               .expectStatusToBe     (-1)
168               .finish               (),
169     testCase().inFolder             ("./10-argument-options")
170               .withParams           ("insensitive --abc 2")
171               .expectStatusToBe     (0)
172               .finish               (),
173     testCase().inFolder             ("./10-argument-options")
174               .withParams           ("insensitive --ABC 2")
175               .expectStatusToBe     (0)
176               .finish               (),
177     testCase().inFolder             ("./10-argument-options")
178               .withParams           ("redefine --abc 2")
179               .expectStatusToBe     (2)
180               .finish               (),
181     testCase().inFolder             ("./10-argument-options")
182               .withParams           ("redefine --abc 2 --abc 1")
183               .expectStatusToBe     (1)
184               .finish               (),
185     testCase().inFolder             ("./10-argument-options")
186               .withParams           ("no-redefine --abc 2")
187               .expectStatusToBe     (0)
188               .finish               (),
189     testCase().inFolder             ("./10-argument-options")
190               .withParams           ("no-redefine --abc 2 --abc 1")
191               .expectStatusToBe     (-1)
192               .finish               (),
193 ];
194  
195 /++ MAIN ++/
196 int main(string[] args)
197 {
198     return (new CommandLineInterface!test()).parseAndExecute(args);
199 }
200 
201 /++ COMMANDS ++/
202 @CommandDefault("Runs all test cases")
203 struct DefaultCommand
204 {
205     int onExecute()
206     {
207         writefln("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
208         const anyfails = runTestSet(TEST_CASES);
209 
210         return anyfails ? -1 : 0;
211     }
212 }
213 
214 @Command("ui", "Runs the test UI")
215 struct UICommand
216 {
217     void onExecute()
218     {
219         auto ui = new UI();
220         ui.run();
221     }
222 }
223 
224 @Command("cleanup", "Runs the cleanup command for all test cases")
225 struct CleanupCommand
226 {
227     void onExecute()
228     {
229         foreach(test; TEST_CASES)
230             runCleanup(test);
231     }
232 }
233 
234 /++ FUNCS ++/
235 bool runTestSet(TestCase[] testSet)
236 {
237     auto results = new TestResult[testSet.length];
238     foreach(i, testCase; testSet)
239         results[i] = testCase.runTest();
240 
241     if(results.any!(r => !r.passed))
242         writefln("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
243 
244     size_t failedCount;
245     foreach(failed; results.filter!(r => !r.passed))
246     {
247         failedCount++;
248         writefln("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
249 
250         foreach(reason; failed.failedReasons)
251             writefln("\t\t- %s".ansi.fg(Ansi4BitColour.red).to!string, reason);
252     }
253 
254     size_t passedCount = results.length - failedCount;
255     writefln(
256         "\n%s %s, %s %s, %s total tests",
257         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
258         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
259         results.length
260     );
261 
262     return (failedCount != 0);
263 }
264 
265 TestResult runTest(TestCase testCase)
266 {
267     const    results = getBuildAndTestResults(testCase);
268     auto     passed  = true;
269     string[] reasons;
270 
271     void failIf(bool condition, string reason)
272     {
273         if(!condition)
274             return;
275 
276         passed   = false;
277         reasons ~= reason;
278     }
279 
280     failIf(results[0].status != 0, "Build failed.");
281 
282     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
283     // a -1 as expecting -1 or 255.
284     if(testCase.expectedStatus.get(0) != -1)
285         failIf(results[1].status != testCase.expectedStatus.get(0), "Status code is wrong.");
286     else
287         failIf(results[1].status != -1 && results[1].status != 255, "Status code is wrong. (-1 special case)");
288 
289     if(!testCase.outputRegex.isNull)
290         failIf(!results[1].output.matchFirst(testCase.outputRegex.get), "Output doesn't contain a match for the given regex.");
291 
292     if(testCase.allowedToFail)
293     {
294         if(!passed)
295             writeln("Test FAILED (ALLOWED).".ansi.fg(Ansi4BitColour.yellow));
296         passed = true;
297     }
298 
299     if(!passed)
300         writeln("Test FAILED".ansi.fg(Ansi4BitColour.red));
301     else
302         writefln("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
303 
304     return TestResult(passed, reasons, testCase, results[0].status, results[1].status, results[0].output, results[1].output);
305 }
306 
307 // [0] = build result, [1] = test result
308 auto getBuildAndTestResults(TestCase testCase)
309 {
310     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
311     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
312     const RESULT_COLOUR   = Ansi4BitColour.yellow;
313 
314     writefln("");
315     writefln("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
316     writefln("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
317     writefln("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
318     writefln("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
319     writefln("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
320 
321     auto cwd = getcwd();
322     chdir(testCase.folder);
323     scope(exit) chdir(cwd);
324 
325     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
326     {
327         writefln("Cleanup: %s".ansi.fg(Ansi4BitColour.brightBlack).to!string, file);
328         remove(file);
329     }
330 
331     const buildString   = "dub build --compiler=ldc2";
332     const commandString = "\"./test\" " ~ testCase.params;
333     const buildResult   = executeShell(buildString);
334     const result        = executeShell(commandString);
335 
336     writefln("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.status.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
337     writefln("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.status.to!string.ansi.fg(RESULT_COLOUR),      result.output);
338 
339     return [buildResult, result];
340 }
341 
342 void runCleanup(TestCase testCase)
343 {
344     auto cwd = getcwd();
345     chdir(testCase.folder);
346     scope(exit) chdir(cwd);
347     executeShell("dub clean");
348 }
349 
350 final class UI
351 {
352     TextBuffer buffer;
353     Layout layout;
354     size_t selectedTest;
355     size_t buildOffset;
356     size_t buildXOffset;
357     size_t runOffset;
358 
359     static struct Test
360     {
361         enum State
362         {
363             notRan,
364             success,
365             failure
366         }
367         string displayName;
368         State state;
369         int buildStatus;
370         int runStatus;
371         string buildOutput;
372         string runOutput;
373     }
374     Test[] tests;
375 
376     void run()
377     {
378         Console.attach();
379         this.buffer = Console.createTextBuffer();
380         this.layout = Layout(
381             Rect(0, 0, this.buffer.width, this.buffer.height),
382             8, 8
383         );
384 
385         this.tests = TEST_CASES.map!(t => Test(" " ~ t.folder ~ " " ~ t.params ~ " ")).array;
386 
387         while(Console.isAttached)
388         {
389             Console.processEvents((e)
390             {
391                 e.match!(
392                     (ConsoleKeyEvent key) => handleKey(key),
393                     (_){}
394                 );
395             });
396 
397             if(Console.isAttached)
398                 this.draw();
399         }
400     }
401 
402     private void handleKey(ConsoleKeyEvent key)
403     {
404         if(!key.isDown)
405             return;
406 
407         // If I weren't lazy I'd bring the shift and ctrl keys into play, but I'm lazy.
408         if(key.key == ConsoleKey.escape)
409             Console.detach();
410         else if(key.key == ConsoleKey.up && this.selectedTest != 0)
411             this.selectedTest--;
412         else if(key.key == ConsoleKey.down && this.selectedTest < TEST_CASES.length-1)
413             this.selectedTest++;
414         else if(key.key == ConsoleKey.enter)
415             this.runTest(this.selectedTest);
416         else if(key.key == ConsoleKey.home && this.buildOffset != 0)
417             this.buildOffset--;
418         else if(key.key == ConsoleKey.end)
419             this.buildOffset++;
420         else if(key.key == ConsoleKey.insert && this.runOffset != 0)
421             this.runOffset--;
422         else if(key.key == ConsoleKey.del)
423             this.runOffset++;
424         else if(key.key == ConsoleKey.right)
425             this.buildXOffset++;
426         else if(key.key == ConsoleKey.left && this.buildXOffset != 0)
427             this.buildXOffset--;
428         else if(key.key == ConsoleKey.back)
429         {
430             foreach(i; 0..TEST_CASES.length)
431                 this.runTest(i);
432         }
433     }
434 
435     private void runTest(size_t test)
436     {
437         const TestResult result = .runTest(TEST_CASES[test]);
438         this.tests[test].state = (result.passed) ? Test.State.success : Test.State.failure;
439         this.tests[test].buildOutput = result.buildOut;
440         this.tests[test].runOutput = result.runOut;
441     }
442 
443     private void draw()
444     {
445         auto testBlock = BorderWidgetBuilder()
446             .withBackground(AnsiColour(Ansi4BitColour.black))
447             .withForeground(AnsiColour(Ansi4BitColour.yellow))
448             .withBlockArea(Rect(0, 0, 3, 7))
449             .withBorderStyle(BorderStyle.all)
450             .withTitle("TESTS")
451             .withTitleAlignment(Alignment.center)
452             .build();
453 
454         auto resultBlock = BorderWidgetBuilder()
455             .withBackground(AnsiColour(Ansi4BitColour.black))
456             .withForeground(AnsiColour(Ansi4BitColour.yellow))
457             .withBlockArea(Rect(3, 0, 8, 7))
458             .withBorderStyle(BorderStyle.all)
459             .withTitle("RESULT")
460             .withTitleAlignment(Alignment.center)
461             .build();
462 
463         testBlock.render(this.layout, this.buffer);
464         resultBlock.render(this.layout, this.buffer);
465 
466         const testArea = testBlock.innerArea(this.layout);
467         const testLayout = Layout(testArea, testArea.width, testArea.height);
468         foreach(i, test; this.tests[this.selectedTest..$])
469         {
470             const fg = 
471                 (test.state == Test.State.notRan)
472                     ? AnsiColour.init
473                     : (test.state == Test.State.failure)
474                         ? AnsiColour(Ansi4BitColour.red)
475                         : AnsiColour(Ansi4BitColour.green);
476 
477             TextWidgetBuilder()
478                 .withBlockArea(Rect(2, cast(int)i, testArea.width, cast(int)i+1))
479                 .withText(test.displayName)
480                 .withStyle(i == 0 ? AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)).fg(fg) : AnsiStyleSet.init.fg(fg))
481                 .build()
482                 .render(testLayout, this.buffer);
483         }
484 
485         const resultArea   = resultBlock.innerArea(this.layout);
486         const resultLayout = Layout(resultArea, 2, 1);
487         const buildArea    = resultLayout.blockRectToRealRect(Rect(0, 0, 1, 1));
488         const runArea      = resultLayout.blockRectToRealRect(Rect(1, 0, 2, 1));
489         const buildLayout  = Layout(buildArea, 1, 1);
490         const runLayout    = Layout(runArea, 1, 1);
491 
492         auto buildBlock = BorderWidgetBuilder()
493             .withBlockArea(Rect(0, 0, 1, 1))
494             .withBorderStyle(BorderStyle.all)
495             .withTitle("Build")
496             .withTitleAlignment(Alignment.center)
497             .build();
498         buildBlock.render(buildLayout, this.buffer);
499 
500         auto runBlock = BorderWidgetBuilder()
501             .withBlockArea(Rect(0, 0, 1, 1))
502             .withBorderStyle(BorderStyle.all)
503             .withTitle("Run")
504             .withTitleAlignment(Alignment.center)
505             .build();
506         runBlock.render(runLayout, this.buffer);
507 
508         const buildOutArea      = buildBlock.innerArea(buildLayout);
509         const runOutArea        = runBlock.innerArea(runLayout);
510         const buildOutLayout    = Layout(buildOutArea, 1, buildOutArea.height);
511         const runOutLayout      = Layout(runOutArea, 1, runOutArea.height);
512 
513         foreach(i, line; this.tests[this.selectedTest].buildOutput.lineSplitter.drop(this.buildOffset).enumerate)
514         {
515             TextWidgetBuilder()
516                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
517                 .withText(line[min(this.buildXOffset, line.length)..$])
518                 .build()
519                 .render(buildOutLayout, this.buffer);
520         }
521 
522         foreach(i, line; this.tests[this.selectedTest].runOutput.lineSplitter.drop(this.runOffset).enumerate)
523         {
524             TextWidgetBuilder()
525                 .withBlockArea(Rect(0, cast(int)i, 1, cast(int)i+1))
526                 .withText(line[min(this.buildXOffset, line.length)..$])
527                 .build()
528                 .render(runOutLayout, this.buffer);
529         }
530 
531         ShortcutsWidgetBuilder!7()
532             .withBackground(AnsiColour(Ansi4BitColour.brightBlack))
533             .withKeyStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.blue)))
534             .withDescriptionStyle(AnsiStyleSet.init.bg(AnsiColour(Ansi4BitColour.brightBlack)))
535             .withShortcut(0, "ESC", "Close UI")
536             .withShortcut(1, "↑↓", "Select test")
537             .withShortcut(2, "ENTER", "Run test")
538             .withShortcut(3, "BACKSPACE", "Run all tests")
539             .withShortcut(4, "INS DEL", "Move Build Output")
540             .withShortcut(5, "PGUP PGDN", "Move Run Output")
541             .withShortcut(6, "←→", "Offset Output")
542             .build()
543             .render(this.buffer);
544 
545         this.buffer.refresh();
546     }
547 }