1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
30 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
31 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
32 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
33 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
35 import com.eviware.soapui.model.iface.Attachment;
36 import com.eviware.soapui.model.support.PropertiesMap;
37 import com.eviware.soapui.model.testsuite.TestCase;
38 import com.eviware.soapui.model.testsuite.TestRunContext;
39 import com.eviware.soapui.model.testsuite.TestRunListener;
40 import com.eviware.soapui.model.testsuite.TestRunner;
41 import com.eviware.soapui.model.testsuite.TestStep;
42 import com.eviware.soapui.model.testsuite.TestStepResult;
43 import com.eviware.soapui.model.testsuite.TestSuite;
44 import com.eviware.soapui.model.testsuite.TestRunner.Status;
45 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
46 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
47 import com.eviware.soapui.report.JUnitReportCollector;
48 import com.eviware.soapui.support.Tools;
49
50 /***
51 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
52 * directly from other classes.
53 * <p>
54 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
55 * calling run</p>
56 *
57 * @author Ole.Matzura
58 */
59
60 public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
61 {
62 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
63
64 private String testSuite;
65 private String testCase;
66 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
67 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
68 private List<TestCase> runningTests = new ArrayList<TestCase>();
69 private List<TestCase> failedTests = new ArrayList<TestCase>();
70 private String endpoint;
71 private String domain;
72 private String password;
73 private String username;
74 private String host;
75
76 private int testSuiteCount;
77 private int testCaseCount;
78 private int testStepCount;
79 private int testAssertionCount;
80
81 private boolean printReport;
82 private String outputFolder;
83 private boolean exportAll;
84 private boolean junitReport;
85 private int exportCount;
86 private JUnitReportCollector reportCollector;
87 private String wssPasswordType;
88 private WsdlProject project;
89
90 /***
91 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
92 *
93 * @param args
94 * @throws Exception
95 */
96
97 @SuppressWarnings("static-access")
98 public static void main( String [] args) throws Exception
99 {
100 new SoapUITestCaseRunner().runFromCommandLine( args );
101 }
102
103 protected boolean processCommandLine( CommandLine cmd )
104 {
105 if( cmd.hasOption( "e"))
106 setEndpoint( cmd.getOptionValue( "e" ) );
107
108 if( cmd.hasOption( "s"))
109 setTestSuite( cmd.getOptionValue( "s") );
110
111 if( cmd.hasOption( "c"))
112 setTestCase( cmd.getOptionValue( "c") );
113
114 if( cmd.hasOption( "u"))
115 setUsername( cmd.getOptionValue( "u") );
116
117 if( cmd.hasOption( "p"))
118 setPassword( cmd.getOptionValue( "p") );
119
120 if( cmd.hasOption( "w"))
121 setWssPasswordType( cmd.getOptionValue( "w") );
122
123 if( cmd.hasOption( "d"))
124 setDomain( cmd.getOptionValue( "d") );
125
126 if( cmd.hasOption( "h"))
127 setHost( cmd.getOptionValue( "h") );
128
129 if( cmd.hasOption( "f"))
130 setOutputFolder( cmd.getOptionValue( "f") );
131
132 if( cmd.hasOption( "t"))
133 SoapUI.initSettings( cmd.getOptionValue( "t" ));
134
135 if( cmd.hasOption( "i"))
136 {
137 enableSwingUI();
138 }
139
140 setPrintReport( cmd.hasOption( "r" ) );
141 setExportAll( cmd.hasOption( "a" ) );
142 setJUnitReport( cmd.hasOption( "j" ) );
143
144 return true;
145 }
146
147 protected SoapUIOptions initCommandLineOptions()
148 {
149 SoapUIOptions options = new SoapUIOptions( "testrunner" );
150 options.addOption( "e", true, "Sets the endpoint" );
151 options.addOption( "s", true, "Sets the testsuite" );
152 options.addOption( "c", true, "Sets the testcase" );
153 options.addOption( "u", true, "Sets the username" );
154 options.addOption( "p", true, "Sets the password" );
155 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
156 options.addOption( "i", false, "Enables Swing UI for scripts" );
157 options.addOption( "d", true, "Sets the domain" );
158 options.addOption( "h", true, "Sets the host" );
159 options.addOption( "r", false, "Prints a small summary report" );
160 options.addOption( "f", true, "Sets the output folder to export results to" );
161 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
162 options.addOption( "a", false, "Turns on exporting of all results" );
163 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
164 return options;
165 }
166
167 /***
168 * Add console appender to groovy log
169 */
170
171
172 public void setExportAll(boolean exportAll)
173 {
174 this.exportAll = exportAll;
175 }
176
177 public void setJUnitReport(boolean junitReport)
178 {
179 this.junitReport = junitReport;
180 if (junitReport)
181 reportCollector = new JUnitReportCollector();
182 }
183
184 public void setOutputFolder(String outputFolder)
185 {
186 this.outputFolder = outputFolder;
187 }
188
189 public SoapUITestCaseRunner()
190 {
191 super( SoapUITestCaseRunner.TITLE );
192 }
193
194 public SoapUITestCaseRunner( String title )
195 {
196 super( title );
197 }
198
199 /***
200 * Controls if a short test summary should be printed after the test runs
201 *
202 * @param printReport a flag controlling if a summary should be printed
203 */
204
205 public void setPrintReport(boolean printReport)
206 {
207 this.printReport = printReport;
208 }
209
210 /***
211 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
212 *
213 * @param host the host to use by all requests
214 */
215
216 public void setHost(String host)
217 {
218 this.host = host;
219 }
220
221 /***
222 * Sets the domain to use for any authentications
223 *
224 * @param domain the domain to use for any authentications
225 */
226
227 public void setDomain(String domain)
228 {
229 log.info( "Setting domain to [" + domain + "]" );
230 this.domain = domain;
231 }
232
233 /***
234 * Sets the password to use for any authentications
235 *
236 * @param domain the password to use for any authentications
237 */
238
239 public void setPassword(String password)
240 {
241 log.info( "Setting password to [" + password + "]" );
242 this.password = password;
243 }
244
245 /***
246 * Sets the WSS password-type to use for any authentications. Setting this will result
247 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
248 * the specified username and password.
249 *
250 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
251 */
252
253 public void setWssPasswordType( String wssPasswordType )
254 {
255 this.wssPasswordType = wssPasswordType;
256 }
257
258 /***
259 * Sets the username to use for any authentications
260 *
261 * @param domain the username to use for any authentications
262 */
263
264 public void setUsername(String username)
265 {
266 log.info( "Setting username to [" + username + "]" );
267 this.username = username;
268 }
269
270 /***
271 * Runs the testcases as configured with setXXX methods
272 *
273 * @throws Exception thrown if any tests fail
274 */
275
276 public void run() throws Exception
277 {
278 initGroovyLog();
279
280 if( outputFolder != null )
281 {
282
283 File folder = new File( outputFolder );
284 if( !folder.exists())
285 folder.mkdirs();
286 }
287
288 assertions.clear();
289
290 String projectFile = getProjectFile();
291
292 if( !new File( projectFile ).exists() )
293 throw new Exception( "soapUI project file [" + projectFile + "] not found" );
294
295 project = new WsdlProject( projectFile, null );
296 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
297
298 long startTime = System.nanoTime();
299
300
301 for( int c = 0; c < project.getTestSuiteCount(); c++ )
302 {
303 TestSuite suite = project.getTestSuiteAt( c );
304 for( int i = 0; i < suite.getTestCaseCount(); i++ )
305 {
306 suite.getTestCaseAt( i ).addTestRunListener( this );
307 if (junitReport)
308 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
309 }
310 }
311
312
313 for( int c = 0; c < project.getTestSuiteCount(); c++ )
314 {
315 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
316 {
317 runSuite( project.getTestSuiteAt( c ));
318 testSuiteCount++;
319
320
321 if( !runningTests.isEmpty() )
322 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
323
324 while( !runningTests.isEmpty() )
325 {
326 Thread.sleep( 100 );
327 }
328 }
329 }
330
331 long timeTaken = (System.nanoTime()-startTime)/1000000;
332
333 if( printReport )
334 {
335 printReport( timeTaken );
336 }
337
338 if (junitReport) {
339 exportJUnitReports( reportCollector, outputFolder );
340 }
341
342 if( assertions.size() > 0 || failedTests.size() > 0 )
343 {
344 throwFailureException();
345 }
346 }
347
348 protected void throwFailureException() throws Exception
349 {
350 StringBuffer buf = new StringBuffer();
351
352 for( int c = 0; c < assertions.size(); c++ )
353 {
354 WsdlMessageAssertion assertion = assertions.get( c );
355 WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
356 failedTests.remove( testRequest.getTestCase() );
357
358 buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
359 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
360
361 WsdlTestStepResult result = assertionResults.get( assertion );
362 StringWriter stringWriter = new StringWriter();
363 PrintWriter writer = new PrintWriter( stringWriter );
364 result.writeTo( writer );
365 buf.append( stringWriter.toString() );
366 }
367
368 while( !failedTests.isEmpty() )
369 {
370 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
371 }
372
373 throw new Exception( buf.toString() );
374 }
375
376 public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
377 {
378 collector.saveReports(folder == null ? "" : folder);
379 }
380
381 public void printReport( long timeTaken )
382 {
383 System.out.println();
384 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
385 System.out.println( "-----------------------------" );
386 System.out.println( "Time Taken: " + timeTaken + "ms" );
387 System.out.println( "Total TestSuites: " + testSuiteCount );
388 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
389 System.out.println( "Total TestSteps: " + testStepCount );
390 System.out.println( "Total Request Assertions: " + testAssertionCount );
391 System.out.println( "Total Failed Assertions: " + assertions.size() );
392 System.out.println( "Total Exported Results: " + exportCount );
393 }
394
395 /***
396 * Run tests in the specified TestSuite
397 *
398 * @param suite the TestSuite to run
399 */
400
401 public void runSuite(TestSuite suite)
402 {
403 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
404 long start = System.currentTimeMillis();
405 for( int c = 0; c < suite.getTestCaseCount(); c++ )
406 {
407 String name = suite.getTestCaseAt( c ).getName();
408 if( testCase == null || name.equalsIgnoreCase( testCase ))
409 {
410 runTestCase( suite.getTestCaseAt( c ));
411 }
412 else
413 {
414 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
415 }
416 }
417 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
418 }
419
420 /***
421 * Runs the specified TestCase
422 *
423 * @param testCase the testcase to run
424 */
425
426 private void runTestCase(TestCase testCase)
427 {
428 runningTests.add( testCase );
429 testCase.run( PropertiesMap.EMPTY_MAP, testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
430 }
431
432 /***
433 * Sets the testcase to run
434 *
435 * @param testCase the testcase to run
436 */
437
438 public void setTestCase(String testCase)
439 {
440 log.info( "setting testCase to [" + testCase + "]" );
441 this.testCase = testCase;
442 }
443
444 /***
445 * Sets the endpoint to use for all test requests
446 *
447 * @param endpoint the endpoint to use for all test requests
448 */
449
450 public void setEndpoint(String endpoint)
451 {
452 log.info( "setting test endpoint to [" + endpoint+ "]" );
453 this.endpoint = endpoint.trim();
454 }
455
456 /***
457 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
458 *
459 * @param testSuite the testSuite to run.
460 */
461
462 public void setTestSuite(String testSuite)
463 {
464 log.info( "setting testSuite to [" + testSuite + "]" );
465 this.testSuite = testSuite;
466 }
467
468 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
469 {
470 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
471 }
472
473 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
474 {
475 TestStep currentStep = runContext.getCurrentStep();
476 log.info( "running step [" + currentStep.getName() + "]" );
477
478 if( currentStep instanceof WsdlTestRequestStep )
479 {
480 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
481 if( endpoint != null && endpoint.length() > 0 )
482 {
483 requestStep.getTestRequest().setEndpoint( endpoint );
484 }
485
486 if( host != null && host.length() > 0 )
487 {
488 try
489 {
490 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
491 requestStep.getTestRequest().setEndpoint( ep );
492 }
493 catch (Exception e)
494 {
495 log.error( "Failed to set host on endpoint", e );
496 }
497 }
498
499 if( username != null && username.length() > 0 )
500 {
501 requestStep.getTestRequest().setUsername( username );
502 }
503
504 if( password != null && password.length() > 0 )
505 {
506 requestStep.getTestRequest().setPassword( password );
507 }
508
509 if( domain != null && domain.length() > 0 )
510 {
511 requestStep.getTestRequest().setDomain( domain );
512 }
513
514 if( wssPasswordType != null && wssPasswordType.length() > 0 )
515 {
516 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
517 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
518 }
519 }
520 }
521
522 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
523 {
524 TestStep currentStep = runContext.getCurrentStep();
525
526 if( currentStep instanceof WsdlTestRequestStep )
527 {
528 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
529 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
530 {
531 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
532 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
533 if( assertion.getStatus() == AssertionStatus.FAILED )
534 {
535 log.info( "ASSERTION FAILED -> " + assertion.getErrors());
536 assertions.add( assertion );
537 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
538 }
539
540 testAssertionCount++;
541 }
542 }
543
544 String countPropertyName = currentStep.getName() + " run count";
545 Long count = (Long) runContext.getProperty( countPropertyName );
546 if( count == null )
547 {
548 count = new Long( 0 );
549 }
550
551 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
552
553 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
554 {
555 try
556 {
557 String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
558 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
559 count.longValue() + "-" + result.getStatus();
560
561 String fileName = nameBase + ".txt";
562 if( outputFolder != null )
563 {
564 fileName = outputFolder + File.separator + fileName;
565 }
566
567 if( result.getStatus() == TestStepStatus.FAILED )
568 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
569
570 PrintWriter writer = new PrintWriter(fileName);
571 result.writeTo(writer);
572 writer.close();
573
574
575 if( result instanceof WsdlTestRequestStepResult )
576 {
577 Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
578 if( attachments != null && attachments.length > 0 )
579 {
580 for( int c = 0; c < attachments.length; c++ )
581 {
582 fileName = nameBase + "-attachment-" + (c+1) + ".";
583
584 Attachment attachment = attachments[c];
585 String contentType = attachment.getContentType();
586 if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
587 {
588 fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
589 }
590 else
591 {
592 fileName += "dat";
593 }
594
595 if( outputFolder != null )
596 {
597 fileName = outputFolder + File.separator + fileName;
598 }
599
600 FileOutputStream outFile = new FileOutputStream( fileName );
601 Tools.writeAll( outFile, attachment.getInputStream() );
602 outFile.close();
603 }
604 }
605 }
606
607 exportCount++;
608 }
609 catch (Exception e)
610 {
611 log.error( "Error saving failed result: " + e, e );
612 }
613 }
614
615 testStepCount++;
616 }
617
618 public void afterRun(TestRunner testRunner, TestRunContext runContext)
619 {
620 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
621 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
622
623 if( testRunner.getStatus() == Status.FAILED )
624 {
625 failedTests.add( testRunner.getTestCase() );
626 }
627
628 runningTests.remove( testRunner.getTestCase() );
629
630 testCaseCount++;
631 }
632
633 protected WsdlProject getProject()
634 {
635 return project;
636 }
637 }