# -*- coding: utf-8 -*-
"""GRASS Python testing framework module for report generation
Copyright (C) 2014 by the GRASS Development Team
This program is free software under the GNU General Public
License (>=v2). Read the file COPYING that comes with GRASS GIS
for details.
:authors: Vaclav Petras
"""
import os
import datetime
import xml.sax.saxutils as saxutils
import xml.etree.ElementTree as et
import subprocess
import sys
import collections
import re
from .utils import ensure_dir
from .checkers import text_to_keyvalue
if sys.version_info[0] == 2:
from StringIO import StringIO
else:
from io import StringIO
# TODO: change text_to_keyvalue to same sep as here
# TODO: create keyvalue file and move it there together with things from checkers
[docs]def keyvalue_to_text(keyvalue, sep='=', vsep='\n', isep=',',
last_vertical=None):
if not last_vertical:
last_vertical = vsep == '\n'
items = []
for key, value in keyvalue.items():
# TODO: use isep for iterables other than strings
if (not isinstance(value, str)
and isinstance(value, collections.Iterable)):
# TODO: this does not work for list of non-strings
value = isep.join(value)
items.append('{key}{sep}{value}'.format(
key=key, sep=sep, value=value))
text = vsep.join(items)
if last_vertical:
text = text + vsep
return text
[docs]def replace_in_file(file_path, pattern, repl):
"""
:param repl: a repl parameter of ``re.sub()`` function
"""
# using tmp file to store the replaced content
tmp_file_path = file_path + '.tmp'
old_file = open(file_path, 'r')
new_file = open(tmp_file_path, 'w')
for line in old_file:
new_file.write(re.sub(pattern=pattern, string=line, repl=repl))
new_file.close()
old_file.close()
# remove old file since it must not exist for rename/move
os.remove(file_path)
# replace old file by new file
# TODO: this can fail in some (random) cases on MS Windows
os.rename(tmp_file_path, file_path)
[docs]class FileAnonymizer(object):
def __init__(self, paths_to_remove, remove_gisbase=True,
remove_gisdbase=False):
self._paths_to_remove = []
if remove_gisbase:
gisbase = os.environ['GISBASE']
self._paths_to_remove.append(gisbase)
if remove_gisdbase:
# import only when really needed to avoid problems with
# translations when environment is not set properly
import grass.script as gscript
gisdbase = gscript.gisenv()['GISDBASE']
self._paths_to_remove.append(gisdbase)
if paths_to_remove:
self._paths_to_remove.extend(paths_to_remove)
[docs] def anonymize(self, filenames):
# besides GISBASE and test recursion start directory (which is
# supposed to be source root directory or similar) we can also try
# to remove user home directory and GISDBASE
# we suppuse that we run in standard grass session
# TODO: provide more effective implementation
for path in self._paths_to_remove:
for filename in filenames:
path_end = r'[\\/]?'
replace_in_file(filename, path + path_end, '')
[docs]def get_source_url(path, revision, line=None):
"""
:param path: directory or file path relative to remote repository root
:param revision: SVN revision (should be a number)
:param line: line in the file (should be None for directories)
"""
tracurl = 'http://trac.osgeo.org/grass/browser/'
if line:
return '{tracurl}{path}?rev={revision}#L{line}'.format(**locals())
else:
return '{tracurl}{path}?rev={revision}'.format(**locals())
[docs]def html_escape(text):
"""Escape ``'&'``, ``'<'``, and ``'>'`` in a string of data."""
return saxutils.escape(text)
[docs]def html_unescape(text):
"""Unescape ``'&'``, ``'<'``, and ``'>'`` in a string of data."""
return saxutils.unescape(text)
[docs]def color_error_line(line):
if line.startswith('ERROR: '):
# TODO: use CSS class
# ignoring the issue with \n at the end, HTML don't mind
line = '<span style="color: red">' + line + "</span>"
if line.startswith('FAIL: '):
# TODO: use CSS class
# ignoring the issue with \n at the end, HTML don't mind
line = '<span style="color: red">' + line + "</span>"
if line.startswith('WARNING: '):
# TODO: use CSS class
# ignoring the issue with \n at the end, HTML don't mind
line = '<span style="color: blue">' + line + "</span>"
#if line.startswith('Traceback ('):
# line = '<span style="color: red">' + line + "</span>"
return line
[docs]def to_web_path(path):
"""Replace OS dependent path separator with slash.
Path on MS Windows are not usable in links on web. For MS Windows,
this replaces backslash with (forward) slash.
"""
if os.path.sep != '/':
return path.replace(os.path.sep, '/')
else:
return path
[docs]def get_svn_revision():
"""Get SVN revision number
:returns: SVN revision number as string or None if it is
not possible to get
"""
# TODO: here should be starting directory
# but now we are using current as starting
p = subprocess.Popen(['svnversion', '.'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
rc = p.poll()
if not rc:
stdout = stdout.strip()
if stdout.endswith('M'):
stdout = stdout[:-1]
if ':' in stdout:
# the first one is the one of source code
stdout = stdout.split(':')[0]
return stdout
else:
return None
[docs]def get_svn_info():
"""Get important information from ``svn info``
:returns: SVN info as dictionary or None
if it is not possible to obtain it
"""
try:
# TODO: introduce directory, not only current
p = subprocess.Popen(['svn', 'info', '.', '--xml'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
rc = p.poll()
info = {}
if not rc:
root = et.fromstring(stdout)
# TODO: get also date if this make sense
# expecting only one <entry> element
entry = root.find('entry')
info['revision'] = entry.get('revision')
info['url'] = entry.find('url').text
relurl = entry.find('relative-url')
# element which is not found is None
# empty element would be bool(el) == False
if relurl is not None:
relurl = relurl.text
# relative path has ^ at the beginning in SVN version 1.8.8
if relurl.startswith('^'):
relurl = relurl[1:]
else:
# SVN version 1.8.8 supports relative-url but older do not
# so, get relative part from absolute URL
const_url_part = 'https://svn.osgeo.org/grass/'
relurl = info['url'][len(const_url_part):]
info['relative-url'] = relurl
return info
# TODO: add this to svnversion function
except OSError as e:
import errno
# ignore No such file or directory
if e.errno != errno.ENOENT:
raise
return None
[docs]def years_ago(date, years):
# dateutil relative delte would be better but this is more portable
return date - datetime.timedelta(weeks=years * 52)
# TODO: these functions should be called only if we know that svn is installed
# this will simplify the functions, caller must handle it anyway
[docs]def get_svn_path_authors(path, from_date=None):
"""
:returns: a set of authors
"""
if from_date is None:
# this is the SVN default for local copies
revision_range = 'BASE:1'
else:
revision_range = 'BASE:{%s}' % from_date
try:
# TODO: allow also usage of --limit
p = subprocess.Popen(['svn', 'log', '--xml',
'--revision', revision_range, path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
rc = p.poll()
if not rc:
root = et.fromstring(stdout)
# TODO: get also date if this make sense
# expecting only one <entry> element
author_nodes = root.iterfind('*/author')
authors = [n.text for n in author_nodes]
return set(authors)
except OSError as e:
import errno
# ignore No such file or directory
if e.errno != errno.ENOENT:
raise
return None
[docs]def get_html_test_authors_table(directory, tests_authors):
# SVN gives us authors of code together with authors of tests
# so test code authors list also contains authors of tests only
# TODO: don't do this for the top level directories?
tests_authors = set(tests_authors)
no_svn_text = ('<span style="font-size: 60%">'
'Test file authors were not obtained.'
'</span>')
if (not tests_authors
or (len(tests_authors) == 1 and list(tests_authors)[0] == '')):
return '<h3>Code and test authors</h3>' + no_svn_text
from_date = years_ago(datetime.date.today(), years=1)
tested_dir_authors = get_svn_path_authors(directory, from_date)
if tested_dir_authors is not None:
not_testing_authors = tested_dir_authors - tests_authors
else:
no_svn_text = ('<span style="font-size: 60%">'
'Authors cannot be obtained using SVN.'
'</span>')
not_testing_authors = tested_dir_authors = [no_svn_text]
if not not_testing_authors:
not_testing_authors = ['all recent authors contributed tests']
test_authors = (
'<h3>Code and test authors</h3>'
'<p style="font-size: 60%"><em>'
'Note that determination of authors is approximate and only'
' recent code authors are considered.'
'</em></p>'
'<table><tbody>'
'<tr><td>Test authors:</td><td>{file_authors}</td></tr>'
'<tr><td>Authors of tested code:</td><td>{code_authors}</td></tr>'
'<tr><td>Authors owing tests:</td><td>{not_testing}</td></tr>'
'</tbody></table>'
.format(
file_authors=', '.join(sorted(tests_authors)),
code_authors=', '.join(sorted(tested_dir_authors)),
not_testing=', '.join(sorted(not_testing_authors))
))
return test_authors
[docs]class GrassTestFilesMultiReporter(object):
def __init__(self, reporters, forgiving=False):
self.reporters = reporters
self.forgiving = forgiving
[docs] def start(self, results_dir):
# TODO: no directory cleaning (self.clean_before)? now cleaned by caller
# TODO: perhaps only those whoe need it should do it (even multiple times)
# and there is also the delet problem
ensure_dir(os.path.abspath(results_dir))
for reporter in self.reporters:
try:
reporter.start(results_dir)
except AttributeError:
if self.forgiving:
pass
else:
raise
[docs] def finish(self):
for reporter in self.reporters:
try:
reporter.finish()
except AttributeError:
if self.forgiving:
pass
else:
raise
[docs] def start_file_test(self, module):
for reporter in self.reporters:
try:
reporter.start_file_test(module)
except AttributeError:
if self.forgiving:
pass
else:
raise
[docs] def end_file_test(self, **kwargs):
for reporter in self.reporters:
try:
reporter.end_file_test(**kwargs)
except AttributeError:
if self.forgiving:
pass
else:
raise
[docs]class GrassTestFilesCountingReporter(object):
def __init__(self):
self.test_files = None
self.files_fail = None
self.files_pass = None
self.file_pass_per = None
self.file_fail_per = None
self.main_start_time = None
self.main_end_time = None
self.main_time = None
self.file_start_time = None
self.file_end_time = None
self.file_time = None
self._start_file_test_called = False
[docs] def start(self, results_dir):
self.test_files = 0
self.files_fail = 0
self.files_pass = 0
# this might be moved to some report start method
self.main_start_time = datetime.datetime.now()
[docs] def finish(self):
self.main_end_time = datetime.datetime.now()
self.main_time = self.main_end_time - self.main_start_time
assert self.test_files == self.files_fail + self.files_pass
if self.test_files:
self.file_pass_per = 100 * float(self.files_pass) / self.test_files
self.file_fail_per = 100 * float(self.files_fail) / self.test_files
else:
# if no tests were executed, probably something bad happened
# try to report at least something
self.file_pass_per = None
self.file_fail_per = None
[docs] def start_file_test(self, module):
self.file_start_time = datetime.datetime.now()
self._start_file_test_called = True
self.test_files += 1
[docs] def end_file_test(self, returncode, **kwargs):
assert self._start_file_test_called
self.file_end_time = datetime.datetime.now()
self.file_time = self.file_end_time - self.file_start_time
if returncode:
self.files_fail += 1
else:
self.files_pass += 1
self._start_file_test_called = False
[docs]def percent_to_html(percent):
if percent is None:
return '<span style="color: {color}">unknown percentage</span>'
elif percent > 100 or percent < 0:
return "? {:.2f}% ?".format(percent)
elif percent < 40:
color = 'red'
elif percent < 70:
color = 'orange'
else:
color = 'green'
return '<span style="color: {color}">{percent:.0f}%</span>'.format(
percent=percent, color=color)
[docs]def wrap_stdstream_to_html(infile, outfile, module, stream):
before = '<html><body><h1>%s</h1><pre>' % (module.name + ' ' + stream)
after = '</pre></body></html>'
html = open(outfile, 'w')
html.write(before)
with open(infile) as text:
for line in text:
html.write(color_error_line(html_escape(line)))
html.write(after)
html.close()
[docs]def html_file_preview(filename):
before = '<pre>'
after = '</pre>'
if not os.path.isfile(filename):
return '<p style="color: red>File %s does not exist<p>' % filename
size = os.path.getsize(filename)
if not size:
return '<p style="color: red>File %s is empty<p>' % filename
max_size = 10000
html = StringIO()
html.write(before)
if size < max_size:
with open(filename) as text:
for line in text:
html.write(color_error_line(html_escape(line)))
elif size < 10 * max_size:
def tail(filename, n):
return collections.deque(open(filename), n)
html.write('... (lines omitted)\n')
for line in tail(filename, 50):
html.write(color_error_line(html_escape(line)))
else:
return '<p style="color: red>File %s is too large to show<p>' % filename
html.write(after)
return html.getvalue()
[docs]def returncode_to_html_text(returncode):
if returncode:
return '<span style="color: red">FAILED</span>'
else:
# alternatives: SUCCEEDED, passed, OK
return '<span style="color: green">succeeded</span>'
# not used
[docs]def returncode_to_html_sentence(returncode):
if returncode:
return ('<span style="color: red">❌</span>'
' Test failed (return code %d)' % (returncode))
else:
return ('<span style="color: green">✓</span>'
' Test succeeded (return code %d)' % (returncode))
[docs]def returncode_to_success_html_par(returncode):
if returncode:
return ('<p> <span style="color: red">❌</span>'
' Test failed</p>')
else:
return ('<p> <span style="color: green">✓</span>'
' Test succeeded</p>')
[docs]def success_to_html_text(total, successes):
if successes < total:
return '<span style="color: red">FAILED</span>'
elif successes == total:
# alternatives: SUCCEEDED, passed, OK
return '<span style="color: green">succeeded</span>'
else:
return ('<span style="color: red; font-size: 60%">'
'? more successes than total ?</span>')
UNKNOWN_NUMBER_HTML = '<span style="font-size: 60%">unknown</span>'
[docs]def success_to_html_percent(total, successes):
if total:
pass_per = 100 * (float(successes) / total)
pass_per = percent_to_html(pass_per)
else:
pass_per = UNKNOWN_NUMBER_HTML
return pass_per
[docs]class GrassTestFilesHtmlReporter(GrassTestFilesCountingReporter):
unknown_number = UNKNOWN_NUMBER_HTML
def __init__(self, file_anonymizer, main_page_name='index.html'):
super(GrassTestFilesHtmlReporter, self).__init__()
self.main_index = None
self._file_anonymizer = file_anonymizer
self._main_page_name = main_page_name
[docs] def start(self, results_dir):
super(GrassTestFilesHtmlReporter, self).start(results_dir)
# having all variables public although not really part of API
main_page_name = os.path.join(results_dir, self._main_page_name)
self.main_index = open(main_page_name, 'w')
# TODO: this can be moved to the counter class
self.failures = 0
self.errors = 0
self.skipped = 0
self.successes = 0
self.expected_failures = 0
self.unexpected_success = 0
self.total = 0
svn_info = get_svn_info()
if not svn_info:
svn_text = ('<span style="font-size: 60%">'
'SVN revision cannot be obtained'
'</span>')
else:
url = get_source_url(path=svn_info['relative-url'],
revision=svn_info['revision'])
svn_text = ('SVN revision'
' <a href="{url}">'
'{rev}</a>'
).format(url=url, rev=svn_info['revision'])
self.main_index.write('<html><body>'
'<h1>Test results</h1>'
'{time:%Y-%m-%d %H:%M:%S}'
' ({svn})'
'<table>'
'<thead><tr>'
'<th>Tested directory</th>'
'<th>Test file</th>'
'<th>Status</th>'
'<th>Tests</th><th>Successful</td>'
'<th>Failed</th><th>Percent successful</th>'
'</tr></thead><tbody>'.format(
time=self.main_start_time,
svn=svn_text))
[docs] def finish(self):
super(GrassTestFilesHtmlReporter, self).finish()
pass_per = success_to_html_percent(total=self.total,
successes=self.successes)
tfoot = ('<tfoot>'
'<tr>'
'<td>Summary</td>'
'<td>{nfiles} test files</td>'
'<td>{nsper}</td>'
'<td>{total}</td><td>{st}</td><td>{ft}</td><td>{pt}</td>'
'</tr>'
'</tfoot>'.format(
nfiles=self.test_files,
nsper=percent_to_html(self.file_pass_per),
st=self.successes, ft=self.failures + self.errors,
total=self.total, pt=pass_per
))
# this is the second place with this function
# TODO: provide one implementation
def format_percentage(percentage):
if percentage is not None:
return "{nsper:.0f}%".format(nsper=percentage)
else:
return "unknown percentage"
summary_sentence = ('\nExecuted {nfiles} test files in {time:}.'
'\nFrom them'
' {nsfiles} files ({nsper}) were successful'
' and {nffiles} files ({nfper}) failed.\n'
.format(
nfiles=self.test_files,
time=self.main_time,
nsfiles=self.files_pass,
nffiles=self.files_fail,
nsper=format_percentage(self.file_pass_per),
nfper=format_percentage(self.file_fail_per)))
self.main_index.write('<tbody>{tfoot}</table>'
'<p>{summary}</p>'
'</body></html>'
.format(
tfoot=tfoot,
summary=summary_sentence))
self.main_index.close()
[docs] def start_file_test(self, module):
super(GrassTestFilesHtmlReporter, self).start_file_test(module)
self.main_index.flush() # to get previous lines to the report
[docs] def end_file_test(self, module, cwd, returncode, stdout, stderr,
test_summary):
super(GrassTestFilesHtmlReporter, self).end_file_test(
module=module, cwd=cwd, returncode=returncode,
stdout=stdout, stderr=stderr)
# considering others according to total is OK when we more or less
# know that input data make sense (total >= errors + failures)
total = test_summary.get('total', None)
failures = test_summary.get('failures', 0)
errors = test_summary.get('errors', 0)
# Python unittest TestResult class is reporting success for no
# errors or failures, so skipped, expected failures and unexpected
# success are ignored
# but successful tests are only total - the others
# TODO: add success counter to GrassTestResult base class
skipped = test_summary.get('skipped', 0)
expected_failures = test_summary.get('expected_failures', 0)
unexpected_successes = test_summary.get('unexpected_successes', 0)
successes = test_summary.get('successes', 0)
self.failures += failures
self.errors += errors
self.skipped += skipped
self.expected_failures += expected_failures
self.unexpected_success += unexpected_successes
# zero would be valid here
if total is not None:
# success are only the clear ones
# percentage is influenced by all
# but putting only failures to table
self.successes += successes
self.total += total
# this will handle zero
pass_per = success_to_html_percent(total=total,
successes=successes)
else:
total = successes = pass_per = self.unknown_number
bad_ones = failures + errors
self.main_index.write(
'<tr><td>{d}</td>'
'<td><a href="{d}/{m}/index.html">{m}</a></td>'
'<td>{status}</td>'
'<td>{ntests}</td><td>{stests}</td>'
'<td>{ftests}</td><td>{ptests}</td>'
'<tr>'.format(
d=to_web_path(module.tested_dir), m=module.name,
status=returncode_to_html_text(returncode),
stests=successes, ftests=bad_ones, ntests=total,
ptests=pass_per))
wrap_stdstream_to_html(infile=stdout,
outfile=os.path.join(cwd, 'stdout.html'),
module=module, stream='stdout')
wrap_stdstream_to_html(infile=stderr,
outfile=os.path.join(cwd, 'stderr.html'),
module=module, stream='stderr')
file_index_path = os.path.join(cwd, 'index.html')
file_index = open(file_index_path, 'w')
file_index.write(
'<!DOCTYPE html><html><head><meta charset="utf-8"></head><body>'
'<h1>{m.name}</h1>'
'<h2>{m.tested_dir} – {m.name}</h2>'
'{status}'
.format(
m=module,
status=returncode_to_success_html_par(returncode),
))
# TODO: include optionally hyper link to test suite
# TODO: file_path is reconstucted in a naive way
# file_path should be stored in the module/test file object and just used here
summary_section = (
'<table><tbody>'
'<tr><td>Test</td><td>{m}</td></tr>'
'<tr><td>Testsuite</td><td>{d}</td></tr>'
'<tr><td>Test file</td><td>{file_path}</td></tr>'
'<tr><td>Status</td><td>{status}</td></tr>'
'<tr><td>Return code</td><td>{rc}</td></tr>'
'<tr><td>Number of tests</td><td>{ntests}</td></tr>'
'<tr><td>Successful tests</td><td>{stests}</td></tr>'
'<tr><td>Failed tests</td><td>{ftests}</td></tr>'
'<tr><td>Percent successful</td><td>{ptests}</td></tr>'
'<tr><td>Test duration</td><td>{dur}</td></tr>'
.format(
d=module.tested_dir, m=module.name,
file_path=os.path.join(module.tested_dir, 'testsuite', module.name + '.' + module.file_type),
status=returncode_to_html_text(returncode),
stests=successes, ftests=bad_ones, ntests=total,
ptests=pass_per, rc=returncode,
dur=self.file_time))
file_index.write(summary_section)
modules = test_summary.get('tested_modules', None)
if modules:
# TODO: replace by better handling of potential lists when parsing
# TODO: create link to module if running in grass or in addons
# alternatively a link to module test summary
if type(modules) is not list:
modules = [modules]
file_index.write(
'<tr><td>Tested modules</td><td>{0}</td></tr>'.format(
', '.join(sorted(set(modules)))))
file_index.write('</tbody></table>')
# here we would have also links to coverage, profiling, ...
#'<li><a href="testcodecoverage/index.html">code coverage</a></li>'
files_section = (
'<h3>Supplementary files</h3>'
'<ul>'
'<li><a href="stdout.html">standard output (stdout)</a></li>'
'<li><a href="stderr.html">standard error output (stderr)</a></li>'
)
file_index.write(files_section)
supplementary_files = test_summary.get('supplementary_files', None)
if supplementary_files:
# this is something we might want to do once for all and not
# risk that it will be done twice or rely that somebody else
# will do it for use
# the solution is perhaps do the multi reporter more grass-specific
# and do all common things, so that other can rely on it and
# moreover something can be shared with other explicitly
# using constructors as seems advantageous for counting
self._file_anonymizer.anonymize(supplementary_files)
for f in supplementary_files:
file_index.write('<li><a href="{f}">{f}</a></li>'.format(f=f))
file_index.write('</ul>')
if returncode:
file_index.write('<h3>Standard error output (stderr)</h3>')
file_index.write(html_file_preview(stderr))
file_index.write('</body></html>')
file_index.close()
if returncode:
pass
# TODO: here we don't have opportunity to write error file
# to stream (stdout/stderr)
# a stream can be added and if not none, we could write
# TODO: document info: additional information to be stored type: dict
# allows overwriting what was collected
[docs]class GrassTestFilesKeyValueReporter(GrassTestFilesCountingReporter):
def __init__(self, info=None):
super(GrassTestFilesKeyValueReporter, self).__init__()
self.result_dir = None
self._info = info
[docs] def start(self, results_dir):
super(GrassTestFilesKeyValueReporter, self).start(results_dir)
# having all variables public although not really part of API
self.result_dir = results_dir
# TODO: this can be moved to the counter class
self.failures = 0
self.errors = 0
self.skipped = 0
self.successes = 0
self.expected_failures = 0
self.unexpected_success = 0
self.total = 0
# TODO: document: tested_dirs is a list and it should fit with names
self.names = []
self.tested_dirs = []
self.files_returncodes = []
# sets (no size specified)
self.modules = set()
self.test_files_authors = set()
[docs] def finish(self):
super(GrassTestFilesKeyValueReporter, self).finish()
# this shoul be moved to some additional meta passed in constructor
svn_info = get_svn_info()
if not svn_info:
svn_revision = ''
else:
svn_revision = svn_info['revision']
summary = {}
summary['files_total'] = self.test_files
summary['files_successes'] = self.files_pass
summary['files_failures'] = self.files_fail
summary['names'] = self.names
summary['tested_dirs'] = self.tested_dirs
# TODO: we don't have a general mechanism for storing any type in text
summary['files_returncodes'] = [str(item)
for item in self.files_returncodes]
# let's use seconds as a universal time delta format
# (there is no standard way how to store time delta as string)
summary['time'] = self.main_time.total_seconds()
status = 'failed' if self.files_fail else 'succeeded'
summary['status'] = status
summary['total'] = self.total
summary['successes'] = self.successes
summary['failures'] = self.failures
summary['errors'] = self.errors
summary['skipped'] = self.skipped
summary['expected_failures'] = self.expected_failures
summary['unexpected_successes'] = self.unexpected_success
summary['test_files_authors'] = self.test_files_authors
summary['tested_modules'] = self.modules
summary['svn_revision'] = svn_revision
# ignoring issues with time zones
summary['timestamp'] = self.main_start_time.strftime('%Y-%m-%d %H:%M:%S')
# TODO: add some general metadata here (passed in constructor)
# add additional information
for key, value in self._info.items():
summary[key] = value
summary_filename = os.path.join(self.result_dir,
'test_keyvalue_result.txt')
with open(summary_filename, 'w') as summary_file:
text = keyvalue_to_text(summary, sep='=', vsep='\n', isep=',')
summary_file.write(text)
[docs] def end_file_test(self, module, cwd, returncode, stdout, stderr,
test_summary):
super(GrassTestFilesKeyValueReporter, self).end_file_test(
module=module, cwd=cwd, returncode=returncode,
stdout=stdout, stderr=stderr)
# TODO: considering others according to total, OK?
# here we are using 0 for total but HTML reporter is using None
total = test_summary.get('total', 0)
failures = test_summary.get('failures', 0)
errors = test_summary.get('errors', 0)
# Python unittest TestResult class is reporting success for no
# errors or failures, so skipped, expected failures and unexpected
# success are ignored
# but successful tests are only total - the others
skipped = test_summary.get('skipped', 0)
expected_failures = test_summary.get('expected_failures', 0)
unexpected_successes = test_summary.get('unexpected_successes', 0)
successes = test_summary.get('successes', 0)
# TODO: move this to counter class and perhaps use aggregation
# rather then inheritance
self.failures += failures
self.errors += errors
self.skipped += skipped
self.expected_failures += expected_failures
self.unexpected_success += unexpected_successes
# TODO: should we test for zero?
if total is not None:
# success are only the clear ones
# percentage is influenced by all
# but putting only failures to table
self.successes += successes
self.total += total
self.files_returncodes.append(returncode)
self.tested_dirs.append(module.tested_dir)
self.names.append(module.name)
modules = test_summary.get('tested_modules', None)
if modules:
# TODO: replace by better handling of potential lists when parsing
# TODO: create link to module if running in grass or in addons
# alternatively a link to module test summary
if type(modules) not in [list, set]:
modules = [modules]
self.modules.update(modules)
test_file_authors = test_summary['test_file_authors']
if type(test_file_authors) not in [list, set]:
test_file_authors = [test_file_authors]
self.test_files_authors.update(test_file_authors)
[docs]class GrassTestFilesTextReporter(GrassTestFilesCountingReporter):
def __init__(self, stream):
super(GrassTestFilesTextReporter, self).__init__()
self._stream = stream
[docs] def finish(self):
super(GrassTestFilesTextReporter, self).finish()
def format_percentage(percentage):
if percentage is not None:
return "{nsper:.0f}%".format(nsper=percentage)
else:
return "unknown percentage"
summary_sentence = ('\nExecuted {nfiles} test files in {time:}.'
'\nFrom them'
' {nsfiles} files ({nsper}) were successful'
' and {nffiles} files ({nfper}) failed.\n'
.format(
nfiles=self.test_files,
time=self.main_time,
nsfiles=self.files_pass,
nffiles=self.files_fail,
nsper=format_percentage(self.file_pass_per),
nfper=format_percentage(self.file_fail_per)))
self._stream.write(summary_sentence)
[docs] def start_file_test(self, module):
super(GrassTestFilesTextReporter, self).start_file_test(module)
self._stream.flush() # to get previous lines to the report
[docs] def end_file_test(self, module, cwd, returncode, stdout, stderr,
test_summary):
super(GrassTestFilesTextReporter, self).end_file_test(
module=module, cwd=cwd, returncode=returncode,
stdout=stdout, stderr=stderr)
if returncode:
self._stream.write(
'{m} from {d} failed'
.format(
d=module.tested_dir,
m=module.name))
num_failed = test_summary.get('failures', 0)
num_failed += test_summary.get('errors', 0)
if num_failed:
if num_failed > 1:
text = ' ({f} tests failed)'
else:
text = ' ({f} test failed)'
self._stream.write(text.format(f=num_failed))
self._stream.write('\n')
# TODO: here we lost the possibility to include also file name
# of the appropriate report
# TODO: there is a quite a lot duplication between this class and html reporter
# TODO: document: do not use it for two reports, it accumulates the results
# TODO: add also keyvalue summary generation?
# wouldn't this conflict with collecting data from report afterwards?
[docs]class TestsuiteDirReporter(object):
def __init__(self, main_page_name, testsuite_page_name='index.html',
top_level_testsuite_page_name=None):
self.main_page_name = main_page_name
self.testsuite_page_name = testsuite_page_name
self.top_level_testsuite_page_name = top_level_testsuite_page_name
# TODO: this might be even a object which could add and validate
self.failures = 0
self.errors = 0
self.skipped = 0
self.successes = 0
self.expected_failures = 0
self.unexpected_successes = 0
self.total = 0
self.testsuites = 0
self.testsuites_successes = 0
self.files = 0
self.files_successes = 0
[docs] def report_for_dir(self, root, directory, test_files):
# TODO: create object from this, so that it can be passed from
# one function to another
# TODO: put the inside of for loop to another function
dir_failures = 0
dir_errors = 0
dir_skipped = 0
dir_successes = 0
dir_expected_failures = 0
dir_unexpected_success = 0
dir_total = 0
test_files_authors = []
file_total = 0
file_successes = 0
page_name = os.path.join(root, directory, self.testsuite_page_name)
if (self.top_level_testsuite_page_name and
os.path.abspath(os.path.join(root, directory))
== os.path.abspath(root)):
page_name = os.path.join(root, self.top_level_testsuite_page_name)
page = open(page_name, 'w')
# TODO: should we use forward slashes also for the HTML because
# it is simpler are more consistent with the rest on MS Windows?
head = (
'<html><body>'
'<h1>{name} testsuite results</h1>'
.format(name=directory))
tests_table_head = (
'<h3>Test files results</h3>'
'<table>'
'<thead><tr>'
'<th>Test file</th><th>Status</th>'
'<th>Tests</th><th>Successful</td>'
'<th>Failed</th><th>Percent successful</th>'
'</tr></thead><tbody>'
)
page.write(head)
page.write(tests_table_head)
for test_file_name in test_files:
# TODO: put keyvalue fine name to constant
summary_filename = os.path.join(root, directory, test_file_name,
'test_keyvalue_result.txt')
#if os.path.exists(summary_filename):
with open(summary_filename, 'r') as keyval_file:
summary = text_to_keyvalue(keyval_file.read(), sep='=')
#else:
# TODO: write else here
# summary = None
if 'total' not in summary:
bad_ones = successes = UNKNOWN_NUMBER_HTML
total = None
else:
bad_ones = summary['failures'] + summary['errors']
successes = summary['successes']
total = summary['total']
self.failures += summary['failures']
self.errors += summary['errors']
self.skipped += summary['skipped']
self.successes += summary['successes']
self.expected_failures += summary['expected_failures']
self.unexpected_successes += summary['unexpected_successes']
self.total += summary['total']
dir_failures += summary['failures']
dir_errors += summary['failures']
dir_skipped += summary['skipped']
dir_successes += summary['successes']
dir_expected_failures += summary['expected_failures']
dir_unexpected_success += summary['unexpected_successes']
dir_total += summary['total']
# TODO: keyvalue method should have types for keys function
# perhaps just the current post processing function is enough
test_file_authors = summary['test_file_authors']
if type(test_file_authors) is not list:
test_file_authors = [test_file_authors]
test_files_authors.extend(test_file_authors)
file_total += 1
file_successes += 0 if summary['returncode'] else 1
pass_per = success_to_html_percent(total=total,
successes=successes)
row = (
'<tr>'
'<td><a href="{f}/index.html">{f}</a></td>'
'<td>{status}</td>'
'<td>{ntests}</td><td>{stests}</td>'
'<td>{ftests}</td><td>{ptests}</td>'
'<tr>'
.format(
f=test_file_name,
status=returncode_to_html_text(summary['returncode']),
stests=successes, ftests=bad_ones, ntests=total,
ptests=pass_per))
page.write(row)
self.testsuites += 1
self.testsuites_successes += 1 if file_successes == file_total else 0
self.files += file_total
self.files_successes += file_successes
dir_pass_per = success_to_html_percent(total=dir_total,
successes=dir_successes)
file_pass_per = success_to_html_percent(total=file_total,
successes=file_successes)
tests_table_foot = (
'</tbody><tfoot><tr>'
'<td>Summary</td>'
'<td>{status}</td>'
'<td>{ntests}</td><td>{stests}</td>'
'<td>{ftests}</td><td>{ptests}</td>'
'</tr></tfoot></table>'
.format(
status=file_pass_per,
stests=dir_successes, ftests=dir_failures + dir_errors,
ntests=dir_total, ptests=dir_pass_per))
page.write(tests_table_foot)
test_authors = get_html_test_authors_table(
directory=directory, tests_authors=test_files_authors)
page.write(test_authors)
page.write('</body></html>')
page.close()
status = success_to_html_text(total=file_total, successes=file_successes)
row = (
'<tr>'
'<td><a href="{d}/{page}">{d}</a></td><td>{status}</td>'
'<td>{nfiles}</td><td>{sfiles}</td><td>{pfiles}</td>'
'<td>{ntests}</td><td>{stests}</td>'
'<td>{ftests}</td><td>{ptests}</td>'
'<tr>'
.format(
d=to_web_path(directory), page=self.testsuite_page_name,
status=status,
nfiles=file_total, sfiles=file_successes, pfiles=file_pass_per,
stests=dir_successes, ftests=dir_failures + dir_errors,
ntests=dir_total, ptests=dir_pass_per))
return row
[docs] def report_for_dirs(self, root, directories):
# TODO: this will need chanages according to potential changes in absolute/relative paths
page_name = os.path.join(root, self.main_page_name)
page = open(page_name, 'w')
head = (
'<html><body>'
'<h1>Testsuites results</h1>'
)
tests_table_head = (
'<table>'
'<thead><tr>'
'<th>Testsuite</th>'
'<th>Status</th>'
'<th>Test files</th><th>Successful</td>'
'<th>Percent successful</th>'
'<th>Tests</th><th>Successful</td>'
'<th>Failed</th><th>Percent successful</th>'
'</tr></thead><tbody>'
)
page.write(head)
page.write(tests_table_head)
for directory, test_files in directories.items():
row = self.report_for_dir(root=root, directory=directory,
test_files=test_files)
page.write(row)
pass_per = success_to_html_percent(total=self.total,
successes=self.successes)
file_pass_per = success_to_html_percent(total=self.files,
successes=self.files_successes)
testsuites_pass_per = success_to_html_percent(
total=self.testsuites, successes=self.testsuites_successes)
tests_table_foot = (
'<tfoot>'
'<tr>'
'<td>Summary</td><td>{status}</td>'
'<td>{nfiles}</td><td>{sfiles}</td><td>{pfiles}</td>'
'<td>{ntests}</td><td>{stests}</td>'
'<td>{ftests}</td><td>{ptests}</td>'
'</tr>'
'</tfoot>'
.format(
status=testsuites_pass_per, nfiles=self.files,
sfiles=self.files_successes, pfiles=file_pass_per,
stests=self.successes, ftests=self.failures + self.errors,
ntests=self.total, ptests=pass_per))
page.write(tests_table_foot)
page.write('</body></html>')
Help Index | Topics Index | Keywords Index | Full Index
© 2003-2019 GRASS Development Team, GRASS GIS 7.2.4svn Reference Manual