-
Notifications
You must be signed in to change notification settings - Fork 295
Refactor tests and metadata #317
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
1e31b4a
feb0b75
47f9a12
3a5eaee
218a71c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -39,50 +39,51 @@ | |
import tempfile | ||
import unittest | ||
|
||
from parameterized import parameterized | ||
from pytest import mark | ||
from testfixtures import compare | ||
|
||
BASE_CMD = sys.executable + ' ' + os.path.abspath('./cpplint.py ') | ||
|
||
def RunShellCommand(cmd: str, args: str, cwd='.'): | ||
""" | ||
executes a command | ||
:param cmd: A string to execute. | ||
:param cwd: from which folder to run. | ||
""" | ||
|
||
stdout_target = subprocess.PIPE | ||
stderr_target = subprocess.PIPE | ||
def run_shell_command(cmd: str, args: str, cwd='.'): | ||
"""Executes a command | ||
|
||
proc = subprocess.Popen(cmd + ' ' + args, | ||
shell=True, | ||
cwd=cwd, | ||
stdout=stdout_target, | ||
stderr=stderr_target) | ||
out, err = proc.communicate() | ||
Args: | ||
cmd: A string to execute. | ||
args: A string with arguments to the command. | ||
cwd: from which folder to run. | ||
""" | ||
cmd, args = cmd.split(), args.split() | ||
proc = subprocess.run(cmd + args, cwd=cwd, capture_output=True) | ||
out, err = proc.stdout, proc.stderr | ||
|
||
# Make output system-agnostic, aka support Windows | ||
if os.sep == '\\': | ||
# TODO: Support scenario with multiple folder inputs | ||
win_path = (os.path.dirname(args.split(' ')[-1]) + '\\').encode() | ||
# TODO: Support scenario with multiple input names | ||
# We currently only support the last arguments as the input name | ||
# to prevent accidentally replacing sed tests. | ||
# Fixing would likely need coding an internal "replace slashes" option for cpplint itself. | ||
win_path = (os.path.dirname(args[-1]) + '\\').encode() | ||
good_path = win_path.replace(b'\\', b'/') | ||
out, err = out.replace(win_path, good_path), err.replace(win_path, good_path) | ||
if os.linesep == '\r\n': | ||
out, err = out.replace(b'\r\n', b'\n'), err.replace(b'\r\n', b'\n') | ||
|
||
# print(err) # to get the output at time of test | ||
return (proc.returncode, out, err) | ||
return proc.returncode, out, err | ||
|
||
|
||
class UsageTest(unittest.TestCase): | ||
|
||
def testHelp(self): | ||
(status, out, err) = RunShellCommand(BASE_CMD, '--help') | ||
(status, out, err) = run_shell_command(BASE_CMD, '--help') | ||
self.assertEqual(0, status) | ||
self.assertEqual(b'', out) | ||
self.assertTrue(err.startswith(b'\nSyntax: cpplint')) | ||
|
||
class TemporaryFolderClassSetup(object): | ||
|
||
class TemporaryFolderClassSetup(unittest.TestCase): | ||
""" | ||
Regression tests: The test starts a filetreewalker scanning for files name *.def | ||
Such files are expected to have as first line the argument | ||
|
@@ -108,7 +109,7 @@ def setUpClass(cls): | |
|
||
@classmethod | ||
def tearDownClass(cls): | ||
if (cls._root): | ||
if cls._root: | ||
# pass | ||
shutil.rmtree(cls._root) | ||
|
||
|
@@ -121,39 +122,39 @@ def get_extra_command_args(self, cwd): | |
"""Override in subclass to add arguments to command""" | ||
return '' | ||
|
||
def checkAllInFolder(self, foldername, expectedDefs): | ||
def check_all_in_folder(self, folder_name, expected_defs): | ||
# uncomment to show complete diff | ||
# self.maxDiff = None | ||
count = 0 | ||
for dirpath, _, fnames in os.walk(foldername): | ||
for dirpath, _, fnames in os.walk(folder_name): | ||
for f in fnames: | ||
if f.endswith('.def'): | ||
count += 1 | ||
self._checkDef(os.path.join(dirpath, f)) | ||
self.assertEqual(count, expectedDefs) | ||
self.check_def(os.path.join(dirpath, f)) | ||
self.assertEqual(count, expected_defs) | ||
|
||
def _checkDef(self, path): | ||
def check_def(self, path): | ||
"""runs command and compares to expected output from def file""" | ||
# self.maxDiff = None # to see full diff | ||
with open(path, 'rb') as filehandle: | ||
datalines = filehandle.readlines() | ||
stdoutLines = int(datalines[2]) | ||
filenames = datalines[0].decode('utf8').strip() | ||
with open(path, 'rb') as file_handle: | ||
data = file_handle.readlines() | ||
stdout_lines = int(data[2]) | ||
filenames = data[0].decode('utf8').strip() | ||
args, _, filenames = filenames.rpartition(" ") | ||
if '*' in filenames: | ||
rel_cwd = os.path.dirname(path) | ||
filenames = ' '.join( | ||
filename[len(rel_cwd)+1:] | ||
filename[len(rel_cwd) + 1:] | ||
for filename in glob.glob(rel_cwd + '/' + filenames) | ||
) | ||
args += ' ' + filenames | ||
self._runAndCheck(path, | ||
args, | ||
int(datalines[1]), | ||
[line.decode('utf8').strip() for line in datalines[3:3 + stdoutLines]], | ||
[line.decode('utf8').strip() for line in datalines[3 + stdoutLines:]]) | ||
self._run_and_compare(path, args, int(data[1]), | ||
[line.decode('utf8').strip() | ||
for line in data[3:3 + stdout_lines]], | ||
[line.decode('utf8').strip() | ||
for line in data[3 + stdout_lines:]]) | ||
|
||
def _runAndCheck( | ||
def _run_and_compare( | ||
self, | ||
definition_file, | ||
args, | ||
|
@@ -165,8 +166,8 @@ def _runAndCheck( | |
cmd = BASE_CMD + self.get_extra_command_args(rel_cwd) | ||
cwd = os.path.join(self._root, rel_cwd) | ||
# command to reproduce, do not forget first two lines have special meaning | ||
print("\ncd " + cwd + " && " + cmd + ' ' + args + " 2> <filename>") | ||
(status, out, err) = RunShellCommand(cmd, args, cwd) | ||
print("\ncd " + cwd + " && " + cmd + ' ' + args + " 2> <filename>") | ||
(status, out, err) = run_shell_command(cmd, args, cwd) | ||
self.assertEqual(expected_status, status, 'bad command status %s' % status) | ||
prefix = 'Failed check in %s comparing to %s for command: %s' % (cwd, definition_file, cmd) | ||
compare('\n'.join(expected_err), err.decode('utf8'), prefix=prefix, show_whitespace=True) | ||
|
@@ -177,29 +178,22 @@ class NoRepoSignatureTests(TemporaryFolderClassSetup, unittest.TestCase): | |
"""runs in a temporary folder (under /tmp in linux) without any .git/.hg/.svn file""" | ||
|
||
def get_extra_command_args(self, cwd): | ||
return (' --repository %s ' % self._root) | ||
|
||
def testChromiumSample(self): | ||
self.checkAllInFolder('./samples/chromium-sample', 1) | ||
|
||
def testVlcSample(self): | ||
self.checkAllInFolder('./samples/vlc-sample', 1) | ||
|
||
def testSillySample(self): | ||
self.checkAllInFolder('./samples/silly-sample', 5) | ||
|
||
def testBoostSample(self): | ||
self.checkAllInFolder('./samples/boost-sample', 4) | ||
|
||
return f' --repository {self._root} ' | ||
|
||
def _test_name_func(fun, _, x): | ||
del fun | ||
return f'test{x.args[0].capitalize()}Sample-{x.args[1]}' | ||
|
||
@parameterized.expand([(folder, case[:-4]) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we still need https://github.com/wolever/parameterized that has had no update in two years, when pytest has https://docs.pytest.org/en/stable/how-to/parametrize.html#parametrize builtin? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That was the first thing I tried. I couldn't get that to work with the unittest-module–subclass setup in this repo. (Plas speling is very importent.) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It is not tested on Python 3.12, 3.13, and pytest > v3 (current is v8). See the open issues and There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Well, this seems like the best thing we have. We could always find an alternative solution later. It doesn't seem like parameterized's actual functionality we use has been impacted by their indeed-worrying neglect yet. (This was a reason I also tried the subtests approach, which works without additional modules but doesn't allow for -k filtering.) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. With #332 I do not see why we need There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we'd have a less messy branch history if we just merge this now while you work on that. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. #322 has now removed all instances of |
||
for folder in ['chromium', 'vlc', 'silly', | ||
'boost', 'protobuf', 'codelite', 'v8'] | ||
for case in os.listdir(f'./samples/{folder}-sample') | ||
if case.endswith('.def')], | ||
name_func=_test_name_func) | ||
@mark.timeout(180) | ||
def testProtobufSample(self): | ||
self.checkAllInFolder('./samples/protobuf-sample', 1) | ||
|
||
def testCodeliteSample(self): | ||
self.checkAllInFolder('./samples/codelite-sample', 1) | ||
def testSamples(self, folder, case): | ||
self.check_def(os.path.join(f'./samples/{folder}-sample', case + '.def')) | ||
|
||
def testV8Sample(self): | ||
self.checkAllInFolder('./samples/v8-sample', 1) | ||
|
||
class GitRepoSignatureTests(TemporaryFolderClassSetup, unittest.TestCase): | ||
"""runs in a temporary folder with .git file""" | ||
|
@@ -210,7 +204,8 @@ def prepare_directory(cls, root): | |
pass | ||
|
||
def testCodeliteSample(self): | ||
self.checkAllInFolder('./samples/codelite-sample', 1) | ||
self.check_all_in_folder('./samples/codelite-sample', 1) | ||
|
||
|
||
class MercurialRepoSignatureTests(TemporaryFolderClassSetup, unittest.TestCase): | ||
"""runs in a temporary folder with .hg file""" | ||
|
@@ -221,7 +216,8 @@ def prepare_directory(cls, root): | |
pass | ||
|
||
def testCodeliteSample(self): | ||
self.checkAllInFolder('./samples/codelite-sample', 1) | ||
self.check_all_in_folder('./samples/codelite-sample', 1) | ||
|
||
|
||
class SvnRepoSignatureTests(TemporaryFolderClassSetup, unittest.TestCase): | ||
"""runs in a temporary folder with .svn file""" | ||
|
@@ -232,7 +228,8 @@ def prepare_directory(cls, root): | |
pass | ||
|
||
def testCodeliteSample(self): | ||
self.checkAllInFolder('./samples/codelite-sample', 1) | ||
self.check_all_in_folder('./samples/codelite-sample', 1) | ||
|
||
|
||
if __name__ == '__main__': | ||
unittest.main() |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why this change?
Can we remove it and use pytest parameterize instead of the unmaintained library?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
https://github.com/pytest-dev/pytest-subtests
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is just a linting thing to show the override hints in PyCharm. It works exactly the same without this change.
I've mentioned that subtests were tried and not filterable in a comment above.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You tried with
pytest-subtests
instead ofwolever/parameterized
?Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think parameterized even has subtests support. Yes, I tried both unitest's own subtests and pytest-subtests. See pytest-dev/pytest-subtests#100.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
#332 has now removed all instances of
unittest.TestCase
.