1 /+dub.sdl: 2 name "test" 3 dependency "jcli" path="../" 4 +/ 5 module test; 6 7 import std, jaster.cli; 8 9 /++ DATA TYPES ++/ 10 struct TestCase 11 { 12 string folder; 13 string params; 14 string[] cleanupFiles; 15 Nullable!int expectedStatus; 16 Nullable!(Regex!char) outputRegex; 17 bool allowedToFail; 18 } 19 20 struct TestCaseBuilder 21 { 22 TestCase testCase; 23 24 TestCaseBuilder inFolder(string folder) 25 { 26 this.testCase.folder = folder; 27 return this; 28 } 29 30 TestCaseBuilder withParams(string params) 31 { 32 this.testCase.params = params; 33 return this; 34 } 35 36 TestCaseBuilder expectStatusToBe(int status) 37 { 38 this.testCase.expectedStatus = status; 39 return this; 40 } 41 42 TestCaseBuilder expectOutputToMatch(string regexString) 43 { 44 this.testCase.outputRegex = regex(regexString); 45 return this; 46 } 47 48 TestCaseBuilder cleanup(string fileName) 49 { 50 this.testCase.cleanupFiles ~= fileName; 51 return this; 52 } 53 54 TestCaseBuilder allowToFail() 55 { 56 this.testCase.allowedToFail = true; 57 return this; 58 } 59 60 TestCase finish() 61 { 62 return this.testCase; 63 } 64 } 65 TestCaseBuilder testCase(){ return TestCaseBuilder(); } 66 67 struct TestResult 68 { 69 bool passed; 70 string[] failedReasons; 71 TestCase testCase; 72 } 73 74 /++ CONFIGURATION ++/ 75 auto TEST_CASES = 76 [ 77 testCase().inFolder ("./00-basic-usage-default-command/") 78 .withParams ("20") 79 .expectStatusToBe (0) 80 .finish (), 81 testCase().inFolder ("./00-basic-usage-default-command/") 82 .withParams ("20 --reverse") 83 .expectStatusToBe (128) 84 .finish (), 85 86 testCase().inFolder ("./01-named-sub-commands/") 87 .withParams ("return 0") 88 .expectStatusToBe (0) 89 .finish (), 90 testCase().inFolder ("./01-named-sub-commands/") 91 .withParams ("r 128") 92 .expectStatusToBe (128) 93 .finish (), 94 95 testCase().inFolder ("./02-shorthand-longhand-args/") 96 .withParams ("return --code 0") 97 .expectStatusToBe (0) 98 .finish (), 99 testCase().inFolder ("./02-shorthand-longhand-args/") 100 .withParams ("r -c=128") 101 .expectStatusToBe (128) 102 .finish (), 103 104 // Class inheritence is broken, but I don't really think I can fix it without compiler changes. 105 testCase().inFolder ("./03-inheritence-base-commands/") 106 .withParams ("add 1 2") 107 .expectStatusToBe (3) 108 .allowToFail () 109 .finish (), 110 testCase().inFolder ("./03-inheritence-base-commands/") 111 .withParams ("add 1 2 --offset=7") 112 .expectStatusToBe (10) 113 .allowToFail () 114 .finish (), 115 116 testCase().inFolder ("./04-custom-arg-binders/") 117 .withParams ("./dub.sdl") 118 .expectStatusToBe (0) 119 .finish (), 120 testCase().inFolder ("./04-custom-arg-binders/") 121 .withParams ("./lalaland.txt") 122 .expectStatusToBe (-1) 123 .finish (), 124 125 testCase().inFolder ("./05-dependency-injection/") 126 .withParams ("dman") 127 .expectStatusToBe (0) 128 .finish (), 129 testCase().inFolder ("./05-dependency-injection/") 130 .withParams ("cman") 131 .expectStatusToBe (128) 132 .finish (), 133 134 testCase().inFolder ("./06-configuration") 135 .withParams ("force exception") 136 .expectOutputToMatch ("$^") // Match nothing 137 .cleanup ("config.json") // Otherwise subsequent runs of this test set won't work. 138 .finish (), 139 testCase().inFolder ("./06-configuration") 140 .withParams ("set verbose true") 141 .expectOutputToMatch ("$^") 142 .finish (), 143 testCase().inFolder ("./06-configuration") 144 .withParams ("set name Bradley") 145 .expectOutputToMatch (".*") // Verbose logging should kick in 146 .finish (), 147 testCase().inFolder ("./06-configuration") 148 .withParams ("force exception") 149 .expectOutputToMatch (".*") // Ditto 150 .finish (), 151 testCase().inFolder ("./06-configuration") 152 .withParams ("greet") 153 .expectOutputToMatch ("Brad") 154 .finish (), 155 156 // Can't use expectOutputToMatch for non-coloured text as it doesn't handle ANSI properly. 157 testCase().inFolder ("./07-text-buffer-table") 158 .withParams ("fixed") 159 .expectStatusToBe (0) 160 .expectOutputToMatch ("Age") 161 .finish (), 162 testCase().inFolder ("./07-text-buffer-table") 163 .withParams ("dynamic") 164 .expectStatusToBe (0) 165 .expectOutputToMatch ("Age") 166 .finish (), 167 168 testCase().inFolder ("./08-arg-binder-validation") 169 .withParams ("20 69") 170 .expectStatusToBe (0) 171 .finish (), 172 testCase().inFolder ("./08-arg-binder-validation") 173 .withParams ("69 69") 174 .expectStatusToBe (-1) 175 .expectOutputToMatch ("Expected number to be even") 176 .finish (), 177 testCase().inFolder ("./08-arg-binder-validation") 178 .withParams ("20 20") 179 .expectStatusToBe (-1) 180 .expectOutputToMatch ("Expected number to be odd") 181 .finish (), 182 183 testCase().inFolder ("./09-raw-unparsed-arg-list") 184 .withParams ("echo -- Some args") 185 .expectStatusToBe (0) 186 .expectOutputToMatch (`Running command 'echo' with arguments \["Some", "args"\]`) 187 .finish (), 188 testCase().inFolder ("./09-raw-unparsed-arg-list") 189 .withParams ("noarg") 190 .expectStatusToBe (0) 191 .expectOutputToMatch (`Running command 'noarg' with arguments \[\]`) 192 .finish (), 193 ]; 194 195 /++ MAIN ++/ 196 int main(string[] args) 197 { 198 return (new CommandLineInterface!test()).parseAndExecute(args); 199 } 200 201 /++ COMMANDS ++/ 202 @CommandDefault("Runs all test cases") 203 struct DefaultCommand 204 { 205 int onExecute() 206 { 207 UserIO.logInfof("Running %s tests.", "ALL".ansi.fg(Ansi4BitColour.green)); 208 const anyFailures = runTestSet(TEST_CASES); 209 210 return anyFailures ? -1 : 0; 211 } 212 } 213 214 @Command("cleanup", "Runs the cleanup command for all test cases") 215 struct CleanupCommand 216 { 217 void onExecute() 218 { 219 foreach(test; TEST_CASES) 220 runCleanup(test); 221 } 222 } 223 224 /++ FUNCS ++/ 225 bool runTestSet(TestCase[] testSet) 226 { 227 auto results = new TestResult[testSet.length]; 228 foreach(i, testCase; testSet) 229 results[i] = testCase.runTest(); 230 231 if(results.any!(r => !r.passed)) 232 UserIO.logInfof("\n\nThe following tests %s:", "FAILED".ansi.fg(Ansi4BitColour.red)); 233 234 size_t failedCount; 235 foreach(failed; results.filter!(r => !r.passed)) 236 { 237 failedCount++; 238 UserIO.logInfof("\t%s", failed.testCase.to!string.ansi.fg(Ansi4BitColour.cyan)); 239 240 foreach(reason; failed.failedReasons) 241 UserIO.logErrorf("\t\t- %s", reason); 242 } 243 244 size_t passedCount = results.length - failedCount; 245 UserIO.logInfof( 246 "\n%s %s, %s %s, %s total tests", 247 passedCount, "PASSED".ansi.fg(Ansi4BitColour.green), 248 failedCount, "FAILED".ansi.fg(Ansi4BitColour.red), 249 results.length 250 ); 251 252 return (failedCount != 0); 253 } 254 255 TestResult runTest(TestCase testCase) 256 { 257 const results = getBuildAndTestResults(testCase); 258 auto passed = true; 259 string[] reasons; 260 261 void failIf(bool condition, string reason) 262 { 263 if(!condition) 264 return; 265 266 passed = false; 267 reasons ~= reason; 268 } 269 270 failIf(results[0].statusCode != 0, "Build failed."); 271 272 // When handling the status code, some terminals allow negative status codes, some don't, so we'll special case expecting 273 // a -1 as expecting -1 or 255. 274 if(testCase.expectedStatus.get(0) != -1) 275 failIf(results[1].statusCode != testCase.expectedStatus.get(0), "Status code is wrong."); 276 else 277 failIf(results[1].statusCode != -1 && results[1].statusCode != 255, "Status code is wrong. (-1 special case)"); 278 279 if(!testCase.outputRegex.isNull) 280 failIf(!results[1].output.match(testCase.outputRegex), "Output doesn't contain a match for the given regex."); 281 282 if(testCase.allowedToFail) 283 { 284 if(!passed) 285 UserIO.logWarningf("Test FAILED (ALLOWED)."); 286 passed = true; 287 } 288 289 if(!passed) 290 UserIO.logErrorf("Test FAILED"); 291 else 292 UserIO.logInfof("%s", "Test PASSED".ansi.fg(Ansi4BitColour.green)); 293 294 return TestResult(passed, reasons, testCase); 295 } 296 297 // [0] = build result, [1] = test result 298 Shell.Result[2] getBuildAndTestResults(TestCase testCase) 299 { 300 const CATEGORY_COLOUR = Ansi4BitColour.magenta; 301 const VALUE_COLOUR = Ansi4BitColour.brightBlue; 302 const RESULT_COLOUR = Ansi4BitColour.yellow; 303 304 UserIO.logInfof(""); 305 UserIO.logInfof("%s", "[Test Case]".ansi.fg(CATEGORY_COLOUR)); 306 UserIO.logInfof("%s: %s", "Folder".ansi.fg(CATEGORY_COLOUR), testCase.folder.ansi.fg(VALUE_COLOUR)); 307 UserIO.logInfof("%s: %s", "Params".ansi.fg(CATEGORY_COLOUR), testCase.params.ansi.fg(VALUE_COLOUR)); 308 UserIO.logInfof("%s: %s", "Status".ansi.fg(CATEGORY_COLOUR), testCase.expectedStatus.get(0).to!string.ansi.fg(RESULT_COLOUR)); 309 UserIO.logInfof("%s: %s", "Regex ".ansi.fg(CATEGORY_COLOUR), testCase.outputRegex.get(regex("N/A")).to!string.ansi.fg(RESULT_COLOUR)); 310 311 Shell.pushLocation(testCase.folder); 312 scope(exit) Shell.popLocation(); 313 314 foreach(file; testCase.cleanupFiles.filter!(f => f.exists)) 315 { 316 UserIO.logTracef("Cleanup: %s", file); 317 remove(file); 318 } 319 320 const buildString = "dub build --compiler=ldc2"; 321 const commandString = "\"./test\" " ~ testCase.params; 322 const buildResult = Shell.execute(buildString); 323 const result = Shell.execute(commandString); 324 325 UserIO.logInfof("\n%s(status: %s):\n%s", "Build output".ansi.fg(CATEGORY_COLOUR), buildResult.statusCode.to!string.ansi.fg(RESULT_COLOUR), buildResult.output); 326 UserIO.logInfof("%s(status: %s):\n%s", "Test output".ansi.fg(CATEGORY_COLOUR), result.statusCode.to!string.ansi.fg(RESULT_COLOUR), result.output); 327 328 return [buildResult, result]; 329 } 330 331 void runCleanup(TestCase testCase) 332 { 333 Shell.pushLocation(testCase.folder); 334 scope(exit) Shell.popLocation(); 335 Shell.execute("dub clean"); 336 }