test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jcli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 }
73 
74 /++ CONFIGURATION ++/
75 auto TEST_CASES = 
76 [
77     testCase().inFolder         ("./00-basic-usage-default-command/")
78               .withParams       ("20")
79               .expectStatusToBe (0)
80               .finish           (),
81     testCase().inFolder         ("./00-basic-usage-default-command/")
82               .withParams       ("20 --reverse")
83               .expectStatusToBe (128)
84               .finish           (),
85     
86     testCase().inFolder         ("./01-named-sub-commands/")
87               .withParams       ("return 0")
88               .expectStatusToBe (0)
89               .finish           (),
90     testCase().inFolder         ("./01-named-sub-commands/")
91               .withParams       ("r 128")
92               .expectStatusToBe (128)
93               .finish           (),
94 
95     testCase().inFolder         ("./02-shorthand-longhand-args/")
96               .withParams       ("return --code 0")
97               .expectStatusToBe (0)
98               .finish           (),
99     testCase().inFolder         ("./02-shorthand-longhand-args/")
100               .withParams       ("r -c=128")
101               .expectStatusToBe (128)
102               .finish           (),
103 
104     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
105     testCase().inFolder         ("./03-inheritence-base-commands/")
106               .withParams       ("add 1 2")
107               .expectStatusToBe (3)
108               .allowToFail      ()
109               .finish           (),
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2 --offset=7")
112               .expectStatusToBe (10)
113               .allowToFail      ()
114               .finish           (),
115 
116     testCase().inFolder         ("./04-custom-arg-binders/")
117               .withParams       ("./dub.sdl")
118               .expectStatusToBe (0)
119               .finish           (),
120     testCase().inFolder         ("./04-custom-arg-binders/")
121               .withParams       ("./lalaland.txt")
122               .expectStatusToBe (-1)
123               .finish           (),
124 
125     testCase().inFolder         ("./05-built-in-binders/")
126               .withParams       ("echo -b -i 2 -f 2.2 -s Hola -e red")
127               .expectStatusToBe (0)
128               .finish           (),
129 
130     testCase().inFolder             ("./08-arg-binder-validation")
131               .withParams           ("20 69")
132               .expectStatusToBe     (0)
133               .finish               (),
134     testCase().inFolder             ("./08-arg-binder-validation")
135               .withParams           ("69 69")
136               .expectStatusToBe     (-1)
137               .expectOutputToMatch  ("Expected number to be even")
138               .finish               (),
139     testCase().inFolder             ("./08-arg-binder-validation")
140               .withParams           ("20 20")
141               .expectStatusToBe     (-1)
142               .expectOutputToMatch  ("Expected number to be odd")
143               .finish               (),
144 
145     testCase().inFolder             ("./09-raw-unparsed-arg-list")
146               .withParams           ("echo -- Some args")
147               .expectStatusToBe     (0)
148               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
149               .finish               (),
150     testCase().inFolder             ("./09-raw-unparsed-arg-list")
151               .withParams           ("noarg")
152               .expectStatusToBe     (0)
153               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
154               .finish               (),
155 
156     testCase().inFolder             ("./10-argument-options")
157               .withParams           ("sensitive --abc 2")
158               .expectStatusToBe     (0)
159               .finish               (),
160     testCase().inFolder             ("./10-argument-options")
161               .withParams           ("sensitive --abC 2")
162               .expectStatusToBe     (-1)
163               .finish               (),
164     testCase().inFolder             ("./10-argument-options")
165               .withParams           ("insensitive --abc 2")
166               .expectStatusToBe     (0)
167               .finish               (),
168     testCase().inFolder             ("./10-argument-options")
169               .withParams           ("insensitive --ABC 2")
170               .expectStatusToBe     (0)
171               .finish               (),
172     testCase().inFolder             ("./10-argument-options")
173               .withParams           ("redefine --abc 2")
174               .expectStatusToBe     (2)
175               .finish               (),
176     testCase().inFolder             ("./10-argument-options")
177               .withParams           ("redefine --abc 2 --abc 1")
178               .expectStatusToBe     (1)
179               .finish               (),
180     testCase().inFolder             ("./10-argument-options")
181               .withParams           ("no-redefine --abc 2")
182               .expectStatusToBe     (0)
183               .finish               (),
184     testCase().inFolder             ("./10-argument-options")
185               .withParams           ("no-redefine --abc 2 --abc 1")
186               .expectStatusToBe     (-1)
187               .finish               (),
188 ];
189  
190 /++ MAIN ++/
191 int main(string[] args)
192 {
193     return (new CommandLineInterface!test()).parseAndExecute(args);
194 }
195 
196 /++ COMMANDS ++/
197 @CommandDefault("Runs all test cases")
198 struct DefaultCommand
199 {
200     int onExecute()
201     {
202         writefln("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
203         const anyfails = runTestSet(TEST_CASES);
204 
205         return anyfails ? -1 : 0;
206     }
207 }
208 
209 @Command("cleanup", "Runs the cleanup command for all test cases")
210 struct CleanupCommand
211 {
212     void onExecute()
213     {
214         foreach(test; TEST_CASES)
215             runCleanup(test);
216     }
217 }
218 
219 /++ FUNCS ++/
220 bool runTestSet(TestCase[] testSet)
221 {
222     auto results = new TestResult[testSet.length];
223     foreach(i, testCase; testSet)
224         results[i] = testCase.runTest();
225 
226     if(results.any!(r => !r.passed))
227         writefln("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
228 
229     size_t failedCount;
230     foreach(failed; results.filter!(r => !r.passed))
231     {
232         failedCount++;
233         writefln("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
234 
235         foreach(reason; failed.failedReasons)
236             writefln("\t\t- %s".ansi.fg(Ansi4BitColour.red).to!string, reason);
237     }
238 
239     size_t passedCount = results.length - failedCount;
240     writefln(
241         "\n%s %s, %s %s, %s total tests",
242         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
243         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
244         results.length
245     );
246 
247     return (failedCount != 0);
248 }
249 
250 TestResult runTest(TestCase testCase)
251 {
252     const    results = getBuildAndTestResults(testCase);
253     auto     passed  = true;
254     string[] reasons;
255 
256     void failIf(bool condition, string reason)
257     {
258         if(!condition)
259             return;
260 
261         passed   = false;
262         reasons ~= reason;
263     }
264 
265     failIf(results[0].status != 0, "Build failed.");
266 
267     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
268     // a -1 as expecting -1 or 255.
269     if(testCase.expectedStatus.get(0) != -1)
270         failIf(results[1].status != testCase.expectedStatus.get(0), "Status code is wrong.");
271     else
272         failIf(results[1].status != -1 && results[1].status != 255, "Status code is wrong. (-1 special case)");
273 
274     if(!testCase.outputRegex.isNull)
275         failIf(!results[1].output.matchFirst(testCase.outputRegex.get), "Output doesn't contain a match for the given regex.");
276 
277     if(testCase.allowedToFail)
278     {
279         if(!passed)
280             writeln("Test FAILED (ALLOWED).".ansi.fg(Ansi4BitColour.yellow));
281         passed = true;
282     }
283 
284     if(!passed)
285         writeln("Test FAILED".ansi.fg(Ansi4BitColour.red));
286     else
287         writefln("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
288 
289     return TestResult(passed, reasons, testCase);
290 }
291 
292 // [0] = build result, [1] = test result
293 auto getBuildAndTestResults(TestCase testCase)
294 {
295     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
296     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
297     const RESULT_COLOUR   = Ansi4BitColour.yellow;
298 
299     writefln("");
300     writefln("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
301     writefln("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
302     writefln("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
303     writefln("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
304     writefln("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
305 
306     auto cwd = getcwd();
307     chdir(testCase.folder);
308     scope(exit) chdir(cwd);
309 
310     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
311     {
312         writefln("Cleanup: %s".ansi.fg(Ansi4BitColour.brightBlack).to!string, file);
313         remove(file);
314     }
315 
316     const buildString   = "dub build --compiler=ldc2";
317     const commandString = "\"./test\" " ~ testCase.params;
318     const buildResult   = executeShell(buildString);
319     const result        = executeShell(commandString);
320 
321     writefln("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.status.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
322     writefln("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.status.to!string.ansi.fg(RESULT_COLOUR),      result.output);
323 
324     return [buildResult, result];
325 }
326 
327 void runCleanup(TestCase testCase)
328 {
329     auto cwd = getcwd();
330     chdir(testCase.folder);
331     scope(exit) chdir(cwd);
332     executeShell("dub clean");
333 }