test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jaster.cli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 }
73 
74 /++ CONFIGURATION ++/
75 auto TEST_CASES = 
76 [
77     testCase().inFolder         ("./00-basic-usage-default-command/")
78               .withParams       ("20")
79               .expectStatusToBe (0)
80               .finish           (),
81     testCase().inFolder         ("./00-basic-usage-default-command/")
82               .withParams       ("20 --reverse")
83               .expectStatusToBe (128)
84               .finish           (),
85     
86     testCase().inFolder         ("./01-named-sub-commands/")
87               .withParams       ("return 0")
88               .expectStatusToBe (0)
89               .finish           (),
90     testCase().inFolder         ("./01-named-sub-commands/")
91               .withParams       ("r 128")
92               .expectStatusToBe (128)
93               .finish           (),
94 
95     testCase().inFolder         ("./02-shorthand-longhand-args/")
96               .withParams       ("return --code 0")
97               .expectStatusToBe (0)
98               .finish           (),
99     testCase().inFolder         ("./02-shorthand-longhand-args/")
100               .withParams       ("r -c=128")
101               .expectStatusToBe (128)
102               .finish           (),
103 
104     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
105     testCase().inFolder         ("./03-inheritence-base-commands/")
106               .withParams       ("add 1 2")
107               .expectStatusToBe (3)
108               .allowToFail      ()
109               .finish           (),
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2 --offset=7")
112               .expectStatusToBe (10)
113               .allowToFail      ()
114               .finish           (),
115 
116     testCase().inFolder         ("./04-custom-arg-binders/")
117               .withParams       ("./dub.sdl")
118               .expectStatusToBe (0)
119               .finish           (),
120     testCase().inFolder         ("./04-custom-arg-binders/")
121               .withParams       ("./lalaland.txt")
122               .expectStatusToBe (-1)
123               .finish           (),
124 
125     testCase().inFolder         ("./05-dependency-injection/")
126               .withParams       ("dman")
127               .expectStatusToBe (0)
128               .finish           (),
129     testCase().inFolder         ("./05-dependency-injection/")
130               .withParams       ("cman")
131               .expectStatusToBe (128)
132               .finish           (),
133 
134     testCase().inFolder             ("./06-configuration")
135               .withParams           ("force exception")
136               .expectOutputToMatch  ("$^")          // Match nothing
137               .cleanup              ("config.json") // Otherwise subsequent runs of this test set won't work.
138               .finish               (),
139     testCase().inFolder             ("./06-configuration")
140               .withParams           ("set verbose true")
141               .expectOutputToMatch  ("$^")
142               .finish               (),
143     testCase().inFolder             ("./06-configuration")
144               .withParams           ("set name Bradley")
145               .expectOutputToMatch  (".*") // Verbose logging should kick in
146               .finish               (), 
147     testCase().inFolder             ("./06-configuration")
148               .withParams           ("force exception")
149               .expectOutputToMatch  (".*") // Ditto
150               .finish               (),
151     testCase().inFolder             ("./06-configuration")
152               .withParams           ("greet")
153               .expectOutputToMatch  ("Brad")
154               .finish               (),
155 
156     // Can't use expectOutputToMatch for non-coloured text as it doesn't handle ANSI properly.
157     testCase().inFolder             ("./07-text-buffer-table")
158               .withParams           ("fixed")
159               .expectStatusToBe     (0)
160               .expectOutputToMatch  ("Age")
161               .finish               (),
162     testCase().inFolder             ("./07-text-buffer-table")
163               .withParams           ("dynamic")
164               .expectStatusToBe     (0)
165               .expectOutputToMatch  ("Age")
166               .finish               (),
167 
168     testCase().inFolder             ("./08-arg-binder-validation")
169               .withParams           ("20 69")
170               .expectStatusToBe     (0)
171               .finish               (),
172     testCase().inFolder             ("./08-arg-binder-validation")
173               .withParams           ("69 69")
174               .expectStatusToBe     (-1)
175               .expectOutputToMatch  ("Expected number to be even")
176               .finish               (),
177     testCase().inFolder             ("./08-arg-binder-validation")
178               .withParams           ("20 20")
179               .expectStatusToBe     (-1)
180               .expectOutputToMatch  ("Expected number to be odd")
181               .finish               (),
182 
183     testCase().inFolder             ("./09-raw-unparsed-arg-list")
184               .withParams           ("echo -- Some args")
185               .expectStatusToBe     (0)
186               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
187               .finish               (),
188     testCase().inFolder             ("./09-raw-unparsed-arg-list")
189               .withParams           ("noarg")
190               .expectStatusToBe     (0)
191               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
192               .finish               (),
193 ];
194  
195 /++ MAIN ++/
196 int main(string[] args)
197 {
198     return (new CommandLineInterface!test()).parseAndExecute(args);
199 }
200 
201 /++ COMMANDS ++/
202 @CommandDefault("Runs all test cases")
203 struct DefaultCommand
204 {
205     int onExecute()
206     {
207         UserIO.logInfof("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
208         const anyFailures = runTestSet(TEST_CASES);
209 
210         return anyFailures ? -1 : 0;
211     }
212 }
213 
214 @Command("cleanup", "Runs the cleanup command for all test cases")
215 struct CleanupCommand
216 {
217     void onExecute()
218     {
219         foreach(test; TEST_CASES)
220             runCleanup(test);
221     }
222 }
223 
224 /++ FUNCS ++/
225 bool runTestSet(TestCase[] testSet)
226 {
227     auto results = new TestResult[testSet.length];
228     foreach(i, testCase; testSet)
229         results[i] = testCase.runTest();
230 
231     if(results.any!(r => !r.passed))
232         UserIO.logInfof("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
233 
234     size_t failedCount;
235     foreach(failed; results.filter!(r => !r.passed))
236     {
237         failedCount++;
238         UserIO.logInfof("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
239 
240         foreach(reason; failed.failedReasons)
241             UserIO.logErrorf("\t\t- %s", reason);
242     }
243 
244     size_t passedCount = results.length - failedCount;
245     UserIO.logInfof(
246         "\n%s %s, %s %s, %s total tests",
247         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
248         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
249         results.length
250     );
251 
252     return (failedCount != 0);
253 }
254 
255 TestResult runTest(TestCase testCase)
256 {
257     const    results = getBuildAndTestResults(testCase);
258     auto     passed  = true;
259     string[] reasons;
260 
261     void failIf(bool condition, string reason)
262     {
263         if(!condition)
264             return;
265 
266         passed   = false;
267         reasons ~= reason;
268     }
269 
270     failIf(results[0].statusCode != 0, "Build failed.");
271 
272     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
273     // a -1 as expecting -1 or 255.
274     if(testCase.expectedStatus.get(0) != -1)
275         failIf(results[1].statusCode != testCase.expectedStatus.get(0), "Status code is wrong.");
276     else
277         failIf(results[1].statusCode != -1 && results[1].statusCode != 255, "Status code is wrong. (-1 special case)");
278 
279     if(!testCase.outputRegex.isNull)
280         failIf(!results[1].output.match(testCase.outputRegex), "Output doesn't contain a match for the given regex.");
281 
282     if(testCase.allowedToFail)
283     {
284         if(!passed)
285             UserIO.logWarningf("Test FAILED (ALLOWED).");
286         passed = true;
287     }
288 
289     if(!passed)
290         UserIO.logErrorf("Test FAILED");
291     else
292         UserIO.logInfof("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
293 
294     return TestResult(passed, reasons, testCase);
295 }
296 
297 // [0] = build result, [1] = test result
298 Shell.Result[2] getBuildAndTestResults(TestCase testCase)
299 {
300     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
301     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
302     const RESULT_COLOUR   = Ansi4BitColour.yellow;
303 
304     UserIO.logInfof("");
305     UserIO.logInfof("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
306     UserIO.logInfof("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
307     UserIO.logInfof("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
308     UserIO.logInfof("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
309     UserIO.logInfof("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
310 
311     Shell.pushLocation(testCase.folder);
312     scope(exit) Shell.popLocation();
313 
314     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
315     {
316         UserIO.logTracef("Cleanup: %s", file);
317         remove(file);
318     }
319 
320     const buildString   = "dub build --compiler=ldc2";
321     const commandString = "\"./test\" " ~ testCase.params;
322     const buildResult   = Shell.execute(buildString);
323     const result        = Shell.execute(commandString);
324 
325     UserIO.logInfof("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.statusCode.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
326     UserIO.logInfof("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.statusCode.to!string.ansi.fg(RESULT_COLOUR),      result.output);
327 
328     return [buildResult, result];
329 }
330 
331 void runCleanup(TestCase testCase)
332 {
333     Shell.pushLocation(testCase.folder);
334     scope(exit) Shell.popLocation();
335     Shell.execute("dub clean");
336 }