test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jcli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17     bool                    allowedToFail;
18 }
19 
20 struct TestCaseBuilder
21 {
22     TestCase testCase;
23 
24     TestCaseBuilder inFolder(string folder)
25     {
26         this.testCase.folder = folder;
27         return this;
28     }
29 
30     TestCaseBuilder withParams(string params)
31     {
32         this.testCase.params = params;
33         return this;
34     }
35 
36     TestCaseBuilder expectStatusToBe(int status)
37     {
38         this.testCase.expectedStatus = status;
39         return this;
40     }
41 
42     TestCaseBuilder expectOutputToMatch(string regexString)
43     {
44         this.testCase.outputRegex = regex(regexString);
45         return this;
46     }
47 
48     TestCaseBuilder cleanup(string fileName)
49     {
50         this.testCase.cleanupFiles ~= fileName;
51         return this;
52     }
53 
54     TestCaseBuilder allowToFail()
55     {
56         this.testCase.allowedToFail = true;
57         return this;
58     }
59 
60     TestCase finish()
61     {
62         return this.testCase;
63     }
64 }
65 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
66 
67 struct TestResult
68 {
69     bool     passed;
70     string[] failedReasons;
71     TestCase testCase;
72 }
73 
74 /++ CONFIGURATION ++/
75 auto TEST_CASES = 
76 [
77     testCase().inFolder         ("./00-basic-usage-default-command/")
78               .withParams       ("20")
79               .expectStatusToBe (0)
80               .finish           (),
81     testCase().inFolder         ("./00-basic-usage-default-command/")
82               .withParams       ("20 --reverse")
83               .expectStatusToBe (128)
84               .finish           (),
85     
86     testCase().inFolder         ("./01-named-sub-commands/")
87               .withParams       ("return 0")
88               .expectStatusToBe (0)
89               .finish           (),
90     testCase().inFolder         ("./01-named-sub-commands/")
91               .withParams       ("r 128")
92               .expectStatusToBe (128)
93               .finish           (),
94 
95     testCase().inFolder         ("./02-shorthand-longhand-args/")
96               .withParams       ("return --code 0")
97               .expectStatusToBe (0)
98               .finish           (),
99     testCase().inFolder         ("./02-shorthand-longhand-args/")
100               .withParams       ("r -c=128")
101               .expectStatusToBe (128)
102               .finish           (),
103 
104     // Class inheritence is broken, but I don't really think I can fix it without compiler changes.
105     testCase().inFolder         ("./03-inheritence-base-commands/")
106               .withParams       ("add 1 2")
107               .expectStatusToBe (3)
108               .allowToFail      ()
109               .finish           (),
110     testCase().inFolder         ("./03-inheritence-base-commands/")
111               .withParams       ("add 1 2 --offset=7")
112               .expectStatusToBe (10)
113               .allowToFail      ()
114               .finish           (),
115 
116     testCase().inFolder         ("./04-custom-arg-binders/")
117               .withParams       ("./dub.sdl")
118               .expectStatusToBe (0)
119               .finish           (),
120     testCase().inFolder         ("./04-custom-arg-binders/")
121               .withParams       ("./lalaland.txt")
122               .expectStatusToBe (-1)
123               .finish           (),
124 
125     testCase().inFolder             ("./08-arg-binder-validation")
126               .withParams           ("20 69")
127               .expectStatusToBe     (0)
128               .finish               (),
129     testCase().inFolder             ("./08-arg-binder-validation")
130               .withParams           ("69 69")
131               .expectStatusToBe     (-1)
132               .expectOutputToMatch  ("Expected number to be even")
133               .finish               (),
134     testCase().inFolder             ("./08-arg-binder-validation")
135               .withParams           ("20 20")
136               .expectStatusToBe     (-1)
137               .expectOutputToMatch  ("Expected number to be odd")
138               .finish               (),
139 
140     testCase().inFolder             ("./09-raw-unparsed-arg-list")
141               .withParams           ("echo -- Some args")
142               .expectStatusToBe     (0)
143               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
144               .finish               (),
145     testCase().inFolder             ("./09-raw-unparsed-arg-list")
146               .withParams           ("noarg")
147               .expectStatusToBe     (0)
148               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
149               .finish               (),
150 
151     testCase().inFolder             ("./10-argument-options")
152               .withParams           ("sensitive --abc 2")
153               .expectStatusToBe     (0)
154               .finish               (),
155     testCase().inFolder             ("./10-argument-options")
156               .withParams           ("sensitive --abC 2")
157               .expectStatusToBe     (-1)
158               .finish               (),
159     testCase().inFolder             ("./10-argument-options")
160               .withParams           ("insensitive --abc 2")
161               .expectStatusToBe     (0)
162               .finish               (),
163     testCase().inFolder             ("./10-argument-options")
164               .withParams           ("insensitive --ABC 2")
165               .expectStatusToBe     (0)
166               .finish               (),
167     testCase().inFolder             ("./10-argument-options")
168               .withParams           ("redefine --abc 2")
169               .expectStatusToBe     (2)
170               .finish               (),
171     testCase().inFolder             ("./10-argument-options")
172               .withParams           ("redefine --abc 2 --abc 1")
173               .expectStatusToBe     (1)
174               .finish               (),
175     testCase().inFolder             ("./10-argument-options")
176               .withParams           ("no-redefine --abc 2")
177               .expectStatusToBe     (0)
178               .finish               (),
179     testCase().inFolder             ("./10-argument-options")
180               .withParams           ("no-redefine --abc 2 --abc 1")
181               .expectStatusToBe     (-1)
182               .finish               (),
183 ];
184  
185 /++ MAIN ++/
186 int main(string[] args)
187 {
188     return (new CommandLineInterface!test()).parseAndExecute(args);
189 }
190 
191 /++ COMMANDS ++/
192 @CommandDefault("Runs all test cases")
193 struct DefaultCommand
194 {
195     int onExecute()
196     {
197         writefln("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
198         const anyfails = runTestSet(TEST_CASES);
199 
200         return anyfails ? -1 : 0;
201     }
202 }
203 
204 @Command("cleanup", "Runs the cleanup command for all test cases")
205 struct CleanupCommand
206 {
207     void onExecute()
208     {
209         foreach(test; TEST_CASES)
210             runCleanup(test);
211     }
212 }
213 
214 /++ FUNCS ++/
215 bool runTestSet(TestCase[] testSet)
216 {
217     auto results = new TestResult[testSet.length];
218     foreach(i, testCase; testSet)
219         results[i] = testCase.runTest();
220 
221     if(results.any!(r => !r.passed))
222         writefln("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
223 
224     size_t failedCount;
225     foreach(failed; results.filter!(r => !r.passed))
226     {
227         failedCount++;
228         writefln("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
229 
230         foreach(reason; failed.failedReasons)
231             writefln("\t\t- %s".ansi.fg(Ansi4BitColour.red).to!string, reason);
232     }
233 
234     size_t passedCount = results.length - failedCount;
235     writefln(
236         "\n%s %s, %s %s, %s total tests",
237         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
238         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
239         results.length
240     );
241 
242     return (failedCount != 0);
243 }
244 
245 TestResult runTest(TestCase testCase)
246 {
247     const    results = getBuildAndTestResults(testCase);
248     auto     passed  = true;
249     string[] reasons;
250 
251     void failIf(bool condition, string reason)
252     {
253         if(!condition)
254             return;
255 
256         passed   = false;
257         reasons ~= reason;
258     }
259 
260     failIf(results[0].status != 0, "Build failed.");
261 
262     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
263     // a -1 as expecting -1 or 255.
264     if(testCase.expectedStatus.get(0) != -1)
265         failIf(results[1].status != testCase.expectedStatus.get(0), "Status code is wrong.");
266     else
267         failIf(results[1].status != -1 && results[1].status != 255, "Status code is wrong. (-1 special case)");
268 
269     if(!testCase.outputRegex.isNull)
270         failIf(!results[1].output.matchFirst(testCase.outputRegex.get), "Output doesn't contain a match for the given regex.");
271 
272     if(testCase.allowedToFail)
273     {
274         if(!passed)
275             writeln("Test FAILED (ALLOWED).".ansi.fg(Ansi4BitColour.yellow));
276         passed = true;
277     }
278 
279     if(!passed)
280         writeln("Test FAILED".ansi.fg(Ansi4BitColour.red));
281     else
282         writefln("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
283 
284     return TestResult(passed, reasons, testCase);
285 }
286 
287 // [0] = build result, [1] = test result
288 auto getBuildAndTestResults(TestCase testCase)
289 {
290     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
291     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
292     const RESULT_COLOUR   = Ansi4BitColour.yellow;
293 
294     writefln("");
295     writefln("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
296     writefln("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
297     writefln("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
298     writefln("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
299     writefln("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
300 
301     auto cwd = getcwd();
302     chdir(testCase.folder);
303     scope(exit) chdir(cwd);
304 
305     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
306     {
307         writefln("Cleanup: %s".ansi.fg(Ansi4BitColour.brightBlack).to!string, file);
308         remove(file);
309     }
310 
311     const buildString   = "dub build --compiler=ldc2";
312     const commandString = "\"./test\" " ~ testCase.params;
313     const buildResult   = executeShell(buildString);
314     const result        = executeShell(commandString);
315 
316     writefln("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.status.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
317     writefln("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.status.to!string.ansi.fg(RESULT_COLOUR),      result.output);
318 
319     return [buildResult, result];
320 }
321 
322 void runCleanup(TestCase testCase)
323 {
324     auto cwd = getcwd();
325     chdir(testCase.folder);
326     scope(exit) chdir(cwd);
327     executeShell("dub clean");
328 }