Source code for grass.gunittest.runner

"""
Testing framework module for running tests in Python unittest fashion

Copyright (C) 2014 by the GRASS Development Team
This program is free software under the GNU General Public
License (>=v2). Read the file COPYING that comes with GRASS GIS
for details.

:authors: Vaclav Petras

File content taken from Python's  ``unittest.runner``, it will be used as
a template. It is not expected that something will left.
"""


import sys
import time

import unittest

__unittest = True


class _WritelnDecorator(object):
    """Used to decorate file-like objects with a handy 'writeln' method"""

    def __init__(self, stream):
        self.stream = stream

    def __getattr__(self, attr):
        if attr in ("stream", "__getstate__"):
            raise AttributeError(attr)
        return getattr(self.stream, attr)

    def writeln(self, arg=None):
        if arg:
            self.write(arg)
        self.write("\n")  # text-mode streams translate to \r\n if needed


[docs]class TestResult(unittest.TestResult): # descriptions and verbosity unused # included for compatibility with unittest's TestResult # where are also unused, so perhaps we can remove them # stream set to None and not included in interface, it would not make sense def __init__(self, stream=None, descriptions=None, verbosity=None): super(TestResult, self).__init__( stream=stream, descriptions=descriptions, verbosity=verbosity ) self.successes = []
[docs] def addSuccess(self, test): super(TestResult, self).addSuccess(test) self.successes.append(test)
# TODO: better would be to pass start at the beginning # alternative is to leave counting time on class # TODO: document: we expect all grass classes to have setTimes # TODO: alternatively, be more forgiving for non-unittest methods
[docs] def setTimes(self, start_time, end_time, time_taken): pass
# TODO: implement this
[docs]class TextTestResult(TestResult): """A test result class that can print formatted text results to a stream. Used by TextTestRunner. """ separator1 = "=" * 70 separator2 = "-" * 70 def __init__(self, stream, descriptions, verbosity): super(TextTestResult, self).__init__( stream=stream, descriptions=descriptions, verbosity=verbosity ) self.stream = _WritelnDecorator(stream) self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions self.start_time = None self.end_time = None self.time_taken = None
[docs] def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: return "\n".join((str(test), doc_first_line)) else: return str(test)
[docs] def startTest(self, test): super(TextTestResult, self).startTest(test) if self.showAll: self.stream.write(self.getDescription(test)) self.stream.write(" ... ") self.stream.flush()
[docs] def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) if self.showAll: self.stream.writeln("ok") elif self.dots: self.stream.write(".") self.stream.flush()
[docs] def addError(self, test, err): super(TextTestResult, self).addError(test, err) if self.showAll: self.stream.writeln("ERROR") elif self.dots: self.stream.write("E") self.stream.flush()
[docs] def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) if self.showAll: self.stream.writeln("FAIL") elif self.dots: self.stream.write("F") self.stream.flush()
[docs] def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) if self.showAll: self.stream.writeln("skipped {0!r}".format(reason)) elif self.dots: self.stream.write("s") self.stream.flush()
[docs] def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) if self.showAll: self.stream.writeln("expected failure") elif self.dots: self.stream.write("x") self.stream.flush()
[docs] def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) if self.showAll: self.stream.writeln("unexpected success") elif self.dots: self.stream.write("u") self.stream.flush()
[docs] def printErrors(self): if self.dots or self.showAll: self.stream.writeln() self.printErrorList("ERROR", self.errors) self.printErrorList("FAIL", self.failures)
[docs] def printErrorList(self, flavour, errors): for test, err in errors: self.stream.writeln(self.separator1) self.stream.writeln("%s: %s" % (flavour, self.getDescription(test))) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err)
[docs] def setTimes(self, start_time, end_time, time_taken): self.start_time = start_time self.end_time = end_time self.time_taken = time_taken
[docs] def stopTestRun(self): super(TextTestResult, self).stopTestRun() self.printErrors() self.stream.writeln(self.separator2) run = self.testsRun self.stream.write("Ran %d test%s" % (run, run != 1 and "s" or "")) if self.time_taken: self.stream.write(" in %.3fs" % (self.time_taken)) self.stream.writeln() expectedFails = unexpectedSuccesses = skipped = 0 results = map( len, (self.expectedFailures, self.unexpectedSuccesses, self.skipped) ) expectedFails, unexpectedSuccesses, skipped = results infos = [] if not self.wasSuccessful(): self.stream.write("FAILED") failed, errored = map(len, (self.failures, self.errors)) if failed: infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) else: self.stream.write("OK") if skipped: infos.append("skipped=%d" % skipped) if expectedFails: infos.append("expected_failures=%d" % expectedFails) if unexpectedSuccesses: infos.append("unexpected_successes=%d" % unexpectedSuccesses) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: self.stream.write("\n")
[docs]class KeyValueTestResult(TestResult): """A test result class that can print formatted text results to a stream. Used by TextTestRunner. """ separator1 = "=" * 70 separator2 = "-" * 70 def __init__(self, stream, test_type=None): super(KeyValueTestResult, self).__init__( stream=stream, descriptions=None, verbosity=None ) self._stream = _WritelnDecorator(stream) self.start_time = None self.end_time = None self.time_taken = None if test_type: self.test_type = test_type else: self.test_type = "not-specified" self._grass_modules = [] self._supplementary_files = []
[docs] def setTimes(self, start_time, end_time, time_taken): self.start_time = start_time self.end_time = end_time self.time_taken = time_taken
[docs] def stopTest(self, test): super(KeyValueTestResult, self).stopTest(test) if hasattr(test, "grass_modules"): self._grass_modules.extend(test.grass_modules) if hasattr(test, "supplementary_files"): self._supplementary_files.extend(test.supplementary_files)
[docs] def stopTestRun(self): super(KeyValueTestResult, self).stopTestRun() infos = [] run = self.testsRun # TODO: name should be included by test file caller # from inspect import getsourcefile # from os.path import abspath # abspath(getsourcefile(lambda _: None)) # not writing name is a good option # infos.append("name=%s" % 'unknown') infos.append("time=%.3fs" % (self.time_taken)) # 'date={rundate}\n' # 'date={runtime}\n' # 'date={start_datetime}\n' # 'date={end_datetime}\n' failed, errored = map(len, (self.failures, self.errors)) succeeded = len(self.successes) results = map( len, (self.expectedFailures, self.unexpectedSuccesses, self.skipped) ) expectedFails, unexpectedSuccesses, skipped = results status = "succeeded" if self.wasSuccessful() else "failed" infos.append("status=%s" % status) # if only errors occur, tests are not counted properly # in unittest, so reconstruct their count here # (using general equation, although now errors would be enough) # alternative is to behave as failed file, i.e. do not # write test details and just write status=failed if not run: run = errored + failed + succeeded infos.append("total=%d" % (run)) infos.append("failures=%d" % failed) infos.append("errors=%d" % errored) infos.append("successes=%d" % succeeded) infos.append("skipped=%d" % skipped) # TODO: document this: if not supported by view, # expected_failures should be counted as failures and vice versa # or both add to skipped as unclear? infos.append("expected_failures=%d" % expectedFails) infos.append("unexpected_successes=%d" % unexpectedSuccesses) # TODO: include each module just once? list good and bad modules? infos.append("tested_modules=%s" % ",".join(self._grass_modules)) infos.append("supplementary_files=%s" % ",".join(self._supplementary_files)) # module, modules?, c, c++?, python # TODO: include also type modules? # TODO: include also C++ code? # TODO: distinguish C and Python modules? infos.append("test_type=%s" % (self.test_type)) self._stream.write("\n".join(infos)) self._stream.write("\n") self._stream.flush()
[docs]class MultiTestResult(TestResult): # descriptions and verbosity unused # included for compatibility with unittest's TestResult # where are also unused, so perhaps we can remove them # stream set to None and not included in interface, it would not make sense def __init__(self, results, forgiving=False, descriptions=None, verbosity=None): super(MultiTestResult, self).__init__( descriptions=descriptions, verbosity=verbosity, stream=None ) self._results = results self._forgiving = forgiving
[docs] def startTest(self, test): super(MultiTestResult, self).startTest(test) for result in self._results: try: result.startTest(test) except AttributeError: if self._forgiving: pass else: raise
[docs] def stopTest(self, test): """Called when the given test has been run""" super(MultiTestResult, self).stopTest(test) for result in self._results: try: result.stopTest(test) except AttributeError: if self._forgiving: pass else: raise
[docs] def addSuccess(self, test): super(MultiTestResult, self).addSuccess(test) for result in self._results: try: result.addSuccess(test) except AttributeError: if self._forgiving: pass else: raise
[docs] def addError(self, test, err): super(MultiTestResult, self).addError(test, err) for result in self._results: try: result.addError(test, err) except AttributeError: if self._forgiving: pass else: raise
[docs] def addFailure(self, test, err): super(MultiTestResult, self).addFailure(test, err) for result in self._results: try: result.addFailure(test, err) except AttributeError: if self._forgiving: pass else: raise
[docs] def addSkip(self, test, reason): super(MultiTestResult, self).addSkip(test, reason) for result in self._results: try: result.addSkip(test, reason) except AttributeError: if self._forgiving: pass else: raise
[docs] def addExpectedFailure(self, test, err): super(MultiTestResult, self).addExpectedFailure(test, err) for result in self._results: try: result.addExpectedFailure(test, err) except AttributeError: if self._forgiving: pass else: raise
[docs] def addUnexpectedSuccess(self, test): super(MultiTestResult, self).addUnexpectedSuccess(test) for result in self._results: try: result.addUnexpectedSuccess(test) except AttributeError: if self._forgiving: pass else: raise
[docs] def printErrors(self): "Called by TestRunner after test run" super(MultiTestResult, self).printErrors() for result in self._results: try: result.printErrors() except AttributeError: if self._forgiving: pass else: raise
[docs] def startTestRun(self): """Called once before any tests are executed. See startTest for a method called before each test. """ super(MultiTestResult, self).startTestRun() for result in self._results: try: result.startTestRun() except AttributeError: if self._forgiving: pass else: raise
[docs] def stopTestRun(self): """Called once after all tests are executed. See stopTest for a method called after each test. """ super(MultiTestResult, self).stopTestRun() for result in self._results: try: result.stopTestRun() except AttributeError: if self._forgiving: pass else: raise
# TODO: better would be to pass start at the beginning # alternative is to leave counting time on class # TODO: document: we expect all grass classes to have setTimes # TODO: alternatively, be more forgiving for non-unittest methods
[docs] def setTimes(self, start_time, end_time, time_taken): for result in self._results: try: result.setTimes(start_time, end_time, time_taken) except AttributeError: if self._forgiving: pass else: raise
[docs]class GrassTestRunner(object): def __init__( self, stream=sys.stderr, descriptions=True, verbosity=1, failfast=False, buffer=False, result=None, ): self.stream = _WritelnDecorator(stream) self.descriptions = descriptions self.verbosity = verbosity self.failfast = failfast self.buffer = buffer self._result = result
[docs] def run(self, test): "Run the given test case or test suite." result = self._result unittest.registerResult(result) result.failfast = self.failfast result.buffer = self.buffer startTime = time.time() startTestRun = getattr(result, "startTestRun", None) if startTestRun is not None: startTestRun() try: test(result) finally: stopTime = time.time() timeTaken = stopTime - startTime setTimes = getattr(result, "setTimes", None) if setTimes is not None: setTimes(startTime, stopTime, timeTaken) stopTestRun = getattr(result, "stopTestRun", None) if stopTestRun is not None: stopTestRun() return result