test source code

1 /+dub.sdl:
2     name "test"
3     dependency "jcli" path="../"
4 +/
5 module test;
6 
7 import std, jaster.cli;
8 
9 /++ DATA TYPES ++/
10 struct TestCase
11 {
12     string                  folder;
13     string                  params;
14     string[]                cleanupFiles;
15     Nullable!int            expectedStatus;
16     Nullable!(Regex!char)   outputRegex;
17 }
18 
19 struct TestCaseBuilder
20 {
21     TestCase testCase;
22 
23     TestCaseBuilder inFolder(string folder)
24     {
25         this.testCase.folder = folder;
26         return this;
27     }
28 
29     TestCaseBuilder withParams(string params)
30     {
31         this.testCase.params = params;
32         return this;
33     }
34 
35     TestCaseBuilder expectStatusToBe(int status)
36     {
37         this.testCase.expectedStatus = status;
38         return this;
39     }
40 
41     TestCaseBuilder expectOutputToMatch(string regexString)
42     {
43         this.testCase.outputRegex = regex(regexString);
44         return this;
45     }
46 
47     TestCaseBuilder cleanup(string fileName)
48     {
49         this.testCase.cleanupFiles ~= fileName;
50         return this;
51     }
52 
53     TestCase finish()
54     {
55         return this.testCase;
56     }
57 }
58 TestCaseBuilder testCase(){ return TestCaseBuilder(); }
59 
60 struct TestResult
61 {
62     bool     passed;
63     string[] failedReasons;
64     TestCase testCase;
65 }
66 
67 /++ CONFIGURATION ++/
68 auto TEST_CASES = 
69 [
70     testCase().inFolder         ("./00-basic-usage-default-command/")
71               .withParams       ("20")
72               .expectStatusToBe (0)
73               .finish           (),
74     testCase().inFolder         ("./00-basic-usage-default-command/")
75               .withParams       ("20 --reverse")
76               .expectStatusToBe (128)
77               .finish           (),
78     
79     testCase().inFolder         ("./01-named-sub-commands/")
80               .withParams       ("return 0")
81               .expectStatusToBe (0)
82               .finish           (),
83     testCase().inFolder         ("./01-named-sub-commands/")
84               .withParams       ("r 128")
85               .expectStatusToBe (128)
86               .finish           (),
87 
88     testCase().inFolder         ("./02-shorthand-longhand-args/")
89               .withParams       ("return --code 0")
90               .expectStatusToBe (0)
91               .finish           (),
92     testCase().inFolder         ("./02-shorthand-longhand-args/")
93               .withParams       ("r -c=128")
94               .expectStatusToBe (128)
95               .finish           (),
96 
97     testCase().inFolder         ("./03-inheritence-base-commands/")
98               .withParams       ("add 1 2")
99               .expectStatusToBe (3)
100               .finish           (),
101     testCase().inFolder         ("./03-inheritence-base-commands/")
102               .withParams       ("add 1 2 --offset=7")
103               .expectStatusToBe (10)
104               .finish           (),
105 
106     testCase().inFolder         ("./04-custom-arg-binders/")
107               .withParams       ("./dub.sdl")
108               .expectStatusToBe (0)
109               .finish           (),
110     testCase().inFolder         ("./04-custom-arg-binders/")
111               .withParams       ("./lalaland.txt")
112               .expectStatusToBe (-1)
113               .finish           (),
114 
115     testCase().inFolder         ("./05-dependency-injection/")
116               .withParams       ("dman")
117               .expectStatusToBe (0)
118               .finish           (),
119     testCase().inFolder         ("./05-dependency-injection/")
120               .withParams       ("cman")
121               .expectStatusToBe (128)
122               .finish           (),
123 
124     testCase().inFolder             ("./06-configuration")
125               .withParams           ("force exception")
126               .expectOutputToMatch  ("$^")          // Match nothing
127               .cleanup              ("config.json") // Otherwise subsequent runs of this test set won't work.
128               .finish               (),
129     testCase().inFolder             ("./06-configuration")
130               .withParams           ("set verbose true")
131               .expectOutputToMatch  ("$^")
132               .finish               (),
133     testCase().inFolder             ("./06-configuration")
134               .withParams           ("set name Bradley")
135               .expectOutputToMatch  (".*") // Verbose logging should kick in
136               .finish               (), 
137     testCase().inFolder             ("./06-configuration")
138               .withParams           ("force exception")
139               .expectOutputToMatch  (".*") // Ditto
140               .finish               (),
141     testCase().inFolder             ("./06-configuration")
142               .withParams           ("greet")
143               .expectOutputToMatch  ("Brad")
144               .finish               (),
145 
146     // Can't use expectOutputToMatch for non-coloured text as it doesn't handle ANSI properly.
147     testCase().inFolder             ("./07-text-buffer-table")
148               .withParams           ("fixed")
149               .expectStatusToBe     (0)
150               .expectOutputToMatch  ("Age")
151               .finish               (),
152     testCase().inFolder             ("./07-text-buffer-table")
153               .withParams           ("dynamic")
154               .expectStatusToBe     (0)
155               .expectOutputToMatch  ("Age")
156               .finish               (),
157 
158     testCase().inFolder             ("./08-arg-binder-validation")
159               .withParams           ("20 69")
160               .expectStatusToBe     (0)
161               .finish               (),
162     testCase().inFolder             ("./08-arg-binder-validation")
163               .withParams           ("69 69")
164               .expectStatusToBe     (-1)
165               .expectOutputToMatch  ("Expected number to be even")
166               .finish               (),
167     testCase().inFolder             ("./08-arg-binder-validation")
168               .withParams           ("20 20")
169               .expectStatusToBe     (-1)
170               .expectOutputToMatch  ("Expected number to be odd")
171               .finish               (),
172 
173     testCase().inFolder             ("./09-raw-unparsed-arg-list")
174               .withParams           ("echo -- Some args")
175               .expectStatusToBe     (0)
176               .expectOutputToMatch  (`Running command 'echo' with arguments \["Some", "args"\]`)
177               .finish               (),
178     testCase().inFolder             ("./09-raw-unparsed-arg-list")
179               .withParams           ("noarg")
180               .expectStatusToBe     (0)
181               .expectOutputToMatch  (`Running command 'noarg' with arguments \[\]`)
182               .finish               (),
183 ];
184  
185 /++ MAIN ++/
186 int main(string[] args)
187 {
188     return (new CommandLineInterface!test()).parseAndExecute(args);
189 }
190 
191 /++ COMMANDS ++/
192 @CommandDefault("Runs all test cases")
193 struct DefaultCommand
194 {
195     int onExecute()
196     {
197         UserIO.logInfof("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green));
198         const anyFailures = runTestSet(TEST_CASES);
199 
200         return anyFailures ? -1 : 0;
201     }
202 }
203 
204 /++ FUNCS ++/
205 bool runTestSet(TestCase[] testSet)
206 {
207     auto results = new TestResult[testSet.length];
208     foreach(i, testCase; testSet)
209         results[i] = testCase.runTest();
210 
211     if(results.any!(r => !r.passed))
212         UserIO.logInfof("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red));
213 
214     size_t failedCount;
215     foreach(failed; results.filter!(r => !r.passed))
216     {
217         failedCount++;
218         UserIO.logInfof("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan));
219 
220         foreach(reason; failed.failedReasons)
221             UserIO.logErrorf("\t\t- %s", reason);
222     }
223 
224     size_t passedCount = results.length - failedCount;
225     UserIO.logInfof(
226         "\n%s %s, %s %s, %s total tests",
227         passedCount, "PASSED".ansi.fg(Ansi4BitColour.green),
228         failedCount, "FAILED".ansi.fg(Ansi4BitColour.red),
229         results.length
230     );
231 
232     return (failedCount != 0);
233 }
234 
235 TestResult runTest(TestCase testCase)
236 {
237     const    results = getBuildAndTestResults(testCase);
238     auto     passed  = true;
239     string[] reasons;
240 
241     void failIf(bool condition, string reason)
242     {
243         if(!condition)
244             return;
245 
246         passed   = false;
247         reasons ~= reason;
248     }
249 
250     failIf(results[0].statusCode != 0, "Build failed.");
251 
252     // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting
253     // a -1 as expecting -1 or 255.
254     if(testCase.expectedStatus.get(0) != -1)
255         failIf(results[1].statusCode != testCase.expectedStatus.get(0), "Status code is wrong.");
256     else
257         failIf(results[1].statusCode != -1 && results[1].statusCode != 255, "Status code is wrong. (-1 special case)");
258 
259     if(!testCase.outputRegex.isNull)
260         failIf(!results[1].output.match(testCase.outputRegex), "Output doesn't contain a match for the given regex.");
261 
262     if(!passed)
263         UserIO.logErrorf("Test FAILED");
264     else
265         UserIO.logInfof("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green));
266 
267     return TestResult(passed, reasons, testCase);
268 }
269 
270 // [0] = build result, [1] = test result
271 Shell.Result[2] getBuildAndTestResults(TestCase testCase)
272 {
273     const CATEGORY_COLOUR = Ansi4BitColour.magenta;
274     const VALUE_COLOUR    = Ansi4BitColour.brightBlue;
275     const RESULT_COLOUR   = Ansi4BitColour.yellow;
276 
277     UserIO.logInfof("");
278     UserIO.logInfof("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR));
279     UserIO.logInfof("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR),  testCase.folder.ansi.fg(VALUE_COLOUR));
280     UserIO.logInfof("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR),  testCase.params.ansi.fg(VALUE_COLOUR));
281     UserIO.logInfof("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR),  testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR));
282     UserIO.logInfof("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR),  testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR));
283 
284     Shell.pushLocation(testCase.folder);
285     scope(exit) Shell.popLocation();
286 
287     foreach(file; testCase.cleanupFiles.filter!(f => f.exists))
288     {
289         UserIO.logTracef("Cleanup: %s", file);
290         remove(file);
291     }
292 
293     const buildString   = "dub build --compiler=ldc2";
294     const commandString = "\"./test\" " ~ testCase.params;
295     const buildResult   = Shell.execute(buildString);
296     const result        = Shell.execute(commandString);
297 
298     UserIO.logInfof("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.statusCode.to!string.ansi.fg(RESULT_COLOUR), buildResult.output);
299     UserIO.logInfof("%s(status: %s):\n%s",   "Test output".ansi.fg(CATEGORY_COLOUR),  result.statusCode.to!string.ansi.fg(RESULT_COLOUR),      result.output);
300 
301     return [buildResult, result];
302 }