diff --git a/.github/workflows/update-changelog.yaml b/.github/workflows/update-changelog.yaml new file mode 100644 index 00000000..4f08d5af --- /dev/null +++ b/.github/workflows/update-changelog.yaml @@ -0,0 +1,34 @@ +# This workflow takes the GitHub release notes an updates the changelog on the +# master branch with the body of the release notes, thereby keeping a log in +# the git repo of the changes. + +name: "Update Changelog" + +on: + workflow_dispatch: + release: + types: [released] + +jobs: + update: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: master + + - name: Update Changelog + uses: stefanzweifel/changelog-updater-action@v1 + with: + release-notes: ${{ github.event.release.body }} + latest-version: ${{ github.event.release.name }} + path-to-changelog: CHANGES.md + + - name: Commit updated CHANGELOG + uses: stefanzweifel/git-auto-commit-action@v4 + with: + branch: master + commit_message: Update CHANGELOG + file_pattern: CHANGES.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f043720c..97cd8689 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,7 +75,7 @@ repos: - id: isort args: ["--sp", "setup.cfg"] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.2.0 hooks: - id: check-ast - id: check-case-conflict diff --git a/CHANGES.md b/CHANGES.md index 3b6a6b39..470ace47 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +0.15 (unreleased) +----------------- + +- An updated hash library will be saved to the results directory when + generating a HTML summary page or when the `--mpl-results-always` flag is + set. A button to download this file is included in the HTML summary. + Various bugfixes, test improvements and documentation updates. [#138] + 0.14 (2022-02-09) ----------------- diff --git a/README.rst b/README.rst index ebdee082..3dd9b742 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ section below. Installing ---------- -This plugin is compatible with Python 2.7, and 3.6 and later, and +This plugin is compatible with Python 3.6 and later, and requires `pytest `__ and `matplotlib `__ to be installed. @@ -86,6 +86,10 @@ can either be specified via the ``--mpl-hash-library=`` command line argument, or via the ``hash_library=`` keyword argument to the ``@pytest.mark.mpl_image_compare`` decorator. +When generating a hash library, the tests will also be run as usual against the +existing hash library specified by ``--mpl-hash-library`` or the keyword argument. +However, generating baseline images will always result in the tests being skipped. + Hybrid Mode: Hashes and Images ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -225,6 +229,12 @@ test (based on the hash library) also shown in the generated summary. This option is applied automatically when generating a HTML summary. +When the ``--mpl-results-always`` option is active, and some hash +comparison tests are performed, a hash library containing all the +result hashes will also be saved to the root of the results directory. +The filename will be extracted from ``--mpl-generate-hash-library``, +``--mpl-hash-library`` or ``hash_library=`` in that order. + Base style ^^^^^^^^^^ diff --git a/RELEASING.md b/RELEASING.md index f3618c72..d91e4bce 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -2,11 +2,10 @@ To make a new release of pytest-mpl follow the following steps: -* Update the chanelog with the date of the release, and ensure that all relevant PRs have changelog entries. -* Push the chanelog to master (via a PR) -* Ensure the sdist and wheel GitHub Actions jobs succeeded on master after the merge of the changelog. +* Ensure the sdist and wheel GitHub Actions jobs succeeded on master after the last merge. * Also ensure that the tarball built has an autogenerated version number from setuptools_scm. -* Tag the new release, using the format `vX.Y.X`. -* Push the tag with `git push upstream master --follow-tags` -* Watch as GitHub actions builds the sdist and universal wheel and pushes them to PyPI for you. +* Write the release notes in the GitHub releases UI, use the autogenerated + notes and tidy up a little. +* Publish the new release, using the format `vX.Y.X`. +* Watch as GitHub actions builds the sdist and universal wheel and pushes them to PyPI for you, and updates CHANGES.md on the master branch. * Enjoy the beverage of your choosing 🍻. diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index d147e57c..b23f50a4 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -34,6 +34,7 @@ import shutil import hashlib import inspect +import logging import tempfile import warnings import contextlib @@ -54,27 +55,6 @@ {actual_path}""" -def _download_file(baseline, filename): - # Note that baseline can be a comma-separated list of URLs that we can - # then treat as mirrors - for base_url in baseline.split(','): - try: - u = urlopen(base_url + filename) - content = u.read() - except Exception as e: - warnings.warn('Downloading {0} failed: {1}'.format(base_url + filename, e)) - else: - break - else: - raise Exception("Could not download baseline image from any of the " - "available URLs") - result_dir = Path(tempfile.mkdtemp()) - filename = result_dir / 'downloaded' - with open(str(filename), 'wb') as tmpfile: - tmpfile.write(content) - return Path(filename) - - def _hash_file(in_stream): """ Hashes an already opened file. @@ -181,8 +161,6 @@ def pytest_configure(config): if generate_dir is not None: if baseline_dir is not None: warnings.warn("Ignoring --mpl-baseline-path since --mpl-generate-path is set") - if results_dir is not None and generate_dir is not None: - warnings.warn("Ignoring --mpl-result-path since --mpl-generate-path is set") if baseline_dir is not None and not baseline_dir.startswith(("https", "http")): baseline_dir = os.path.abspath(baseline_dir) @@ -283,11 +261,23 @@ def __init__(self, self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) self.results_dir.mkdir(parents=True, exist_ok=True) + # Decide what to call the downloadable results hash library + if self.hash_library is not None: + self.results_hash_library_name = self.hash_library.name + else: # Use the first filename encountered in a `hash_library=` kwarg + self.results_hash_library_name = None + # We need global state to store all the hashes generated over the run self._generated_hash_library = {} self._test_results = {} self._test_stats = None + # https://stackoverflow.com/questions/51737378/how-should-i-log-in-my-pytest-plugin + # turn debug prints on only if "-vv" or more passed + level = logging.DEBUG if config.option.verbose > 1 else logging.INFO + logging.basicConfig(level=level) + self.logger = logging.getLogger('pytest-mpl') + def get_compare(self, item): """ Return the mpl_image_compare marker for the given item. @@ -360,6 +350,26 @@ def get_baseline_directory(self, item): return baseline_dir + def _download_file(self, baseline, filename): + # Note that baseline can be a comma-separated list of URLs that we can + # then treat as mirrors + for base_url in baseline.split(','): + try: + u = urlopen(base_url + filename) + content = u.read() + except Exception as e: + self.logger.info(f'Downloading {base_url + filename} failed: {repr(e)}') + else: + break + else: + raise Exception("Could not download baseline image from any of the " + "available URLs") + result_dir = Path(tempfile.mkdtemp()) + filename = result_dir / 'downloaded' + with open(str(filename), 'wb') as tmpfile: + tmpfile.write(content) + return Path(filename) + def obtain_baseline_image(self, item, target_dir): """ Copy the baseline image to our working directory. @@ -374,7 +384,7 @@ def obtain_baseline_image(self, item, target_dir): if baseline_remote: # baseline_dir can be a list of URLs when remote, so we have to # pass base and filename to download - baseline_image = _download_file(baseline_dir, filename) + baseline_image = self._download_file(baseline_dir, filename) else: baseline_image = (baseline_dir / filename).absolute() @@ -390,11 +400,14 @@ def generate_baseline_image(self, item, fig): if not os.path.exists(self.generate_dir): os.makedirs(self.generate_dir) - fig.savefig(str((self.generate_dir / self.generate_filename(item)).absolute()), - **savefig_kwargs) + baseline_filename = self.generate_filename(item) + baseline_path = (self.generate_dir / baseline_filename).absolute() + fig.savefig(str(baseline_path), **savefig_kwargs) close_mpl_figure(fig) + return baseline_path + def generate_image_hash(self, item, fig): """ For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal @@ -435,6 +448,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): if not os.path.exists(baseline_image_ref): summary['status'] = 'failed' + summary['image_status'] = 'missing' error_message = ("Image file not found for comparison test in: \n\t" f"{self.get_baseline_directory(item)}\n" "(This is expected for new tests.)\n" @@ -456,6 +470,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): actual_shape = imread(str(test_image)).shape[:2] if expected_shape != actual_shape: summary['status'] = 'failed' + summary['image_status'] = 'diff' error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image, expected_shape=expected_shape, actual_path=test_image, @@ -467,10 +482,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None): summary['tolerance'] = tolerance if results is None: summary['status'] = 'passed' + summary['image_status'] = 'match' summary['status_msg'] = 'Image comparison passed.' return None else: summary['status'] = 'failed' + summary['image_status'] = 'diff' summary['rms'] = results['rms'] diff_image = (result_dir / 'result-failed-diff.png').absolute() summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix() @@ -496,6 +513,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): compare = self.get_compare(item) savefig_kwargs = compare.kwargs.get('savefig_kwargs', {}) + if not self.results_hash_library_name: + # Use hash library name of current test as results hash library name + self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name + hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None) hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute() @@ -512,14 +533,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): if baseline_hash is None: # hash-missing summary['status'] = 'failed' + summary['hash_status'] = 'missing' summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. " f"Generated hash is {test_hash}.") elif test_hash == baseline_hash: # hash-match hash_comparison_pass = True summary['status'] = 'passed' + summary['hash_status'] = 'match' summary['status_msg'] = 'Test hash matches baseline hash.' else: # hash-diff summary['status'] = 'failed' + summary['hash_status'] = 'diff' summary['status_msg'] = (f"Hash {test_hash} doesn't match hash " f"{baseline_hash} in library " f"{hash_library_filename} for test {hash_name}.") @@ -544,7 +568,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None): except Exception as baseline_error: # Append to test error later baseline_comparison = str(baseline_error) else: # Update main summary - for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']: + for k in ['image_status', 'baseline_image', 'diff_image', + 'rms', 'tolerance', 'result_image']: summary[k] = summary[k] or baseline_summary.get(k) # Append the log from image comparison @@ -597,9 +622,12 @@ def item_function_wrapper(*args, **kwargs): remove_ticks_and_titles(fig) test_name = self.generate_test_name(item) + result_dir = self.make_test_results_dir(item) summary = { 'status': None, + 'image_status': None, + 'hash_status': None, 'status_msg': None, 'baseline_image': None, 'diff_image': None, @@ -614,21 +642,23 @@ def item_function_wrapper(*args, **kwargs): # reference images or simply running the test. if self.generate_dir is not None: summary['status'] = 'skipped' + summary['image_status'] = 'generated' summary['status_msg'] = 'Skipped test, since generating image.' - self.generate_baseline_image(item, fig) - if self.generate_hash_library is None: - self._test_results[str(pathify(test_name))] = summary - pytest.skip("Skipping test, since generating image.") + generate_image = self.generate_baseline_image(item, fig) + if self.results_always: # Make baseline image available in HTML + result_image = (result_dir / "baseline.png").absolute() + shutil.copy(generate_image, result_image) + summary['baseline_image'] = \ + result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: + summary['hash_status'] = 'generated' image_hash = self.generate_image_hash(item, fig) self._generated_hash_library[test_name] = image_hash - summary['result_hash'] = image_hash + summary['baseline_hash'] = image_hash # Only test figures if not generating images if self.generate_dir is None: - result_dir = self.make_test_results_dir(item) - # Compare to hash library if self.hash_library or compare.kwargs.get('hash_library', None): msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) @@ -645,12 +675,15 @@ def item_function_wrapper(*args, **kwargs): for image_type in ['baseline_image', 'diff_image', 'result_image']: summary[image_type] = None # image no longer exists else: - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary pytest.fail(msg, pytrace=False) close_mpl_figure(fig) - self._test_results[str(pathify(test_name))] = summary + self._test_results[test_name] = summary + + if summary['status'] == 'skipped': + pytest.skip(summary['status_msg']) if item.cls is not None: setattr(item.cls, item.function.__name__, item_function_wrapper) @@ -667,21 +700,36 @@ def pytest_unconfigure(self, config): """ Save out the hash library at the end of the run. """ + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: hash_library_path = Path(config.rootdir) / self.generate_hash_library hash_library_path.parent.mkdir(parents=True, exist_ok=True) with open(hash_library_path, "w") as fp: json.dump(self._generated_hash_library, fp, indent=2) + if self.results_always: # Make accessible in results directory + # Use same name as generated + result_hash_library = self.results_dir / hash_library_path.name + shutil.copy(hash_library_path, result_hash_library) + elif self.results_always and self.results_hash_library_name: + result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() + if v['result_hash']} + if len(result_hashes) > 0: # At least one hash comparison test + with open(result_hash_library, "w") as fp: + json.dump(result_hashes, fp, indent=2) if self.generate_summary: + kwargs = {} if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if result_hash_library.exists(): # link to it in the HTML + kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: - summary = generate_summary_html(self._test_results, self.results_dir) + summary = generate_summary_html(self._test_results, self.results_dir, **kwargs) print(f"A summary of the failed tests can be found at: {summary}") if 'basic-html' in self.generate_summary: - summary = generate_summary_basic_html(self._test_results, self.results_dir) + summary = generate_summary_basic_html(self._test_results, self.results_dir, + **kwargs) print(f"A summary of the failed tests can be found at: {summary}") diff --git a/pytest_mpl/summary/html.py b/pytest_mpl/summary/html.py index 858cfe21..78fbcc19 100644 --- a/pytest_mpl/summary/html.py +++ b/pytest_mpl/summary/html.py @@ -26,26 +26,12 @@ class Results: def __init__(self, results, title="Image comparison"): self.title = title # HTML - # If any baseline images or baseline hashes are present, - # assume all results should have them - self.warn_missing = {'baseline_image': False, 'baseline_hash': False} - for key in self.warn_missing.keys(): - for result in results.values(): - if result[key] is not None: - self.warn_missing[key] = True - break - - # Additional <body> classes - self.classes = [] - if self.warn_missing['baseline_hash'] is False: - self.classes += ['no-hash-test'] - # Generate sorted list of results self.cards = [] pad = len(str(len(results.items()))) # maximum length of a result index for collect_n, (name, item) in enumerate(results.items()): card_id = str(collect_n).zfill(pad) # zero pad for alphanumerical sorting - self.cards += [Result(name, item, card_id, self.warn_missing)] + self.cards += [Result(name, item, card_id)] self.cards = sorted(self.cards, key=lambda i: i.indexes['status'], reverse=True) @cached_property @@ -66,6 +52,22 @@ def statistics(self): stats['skipped'] += 1 return stats + @cached_property + def image_comparison(self): + """Whether at least one image comparison test or generation was performed.""" + for result in self.cards: + if result.image_status: + return True + return False + + @cached_property + def hash_comparison(self): + """Whether at least one hash comparison test or generation was performed.""" + for result in self.cards: + if result.hash_status: + return True + return False + class Result: """ @@ -81,20 +83,14 @@ class Result: id : str The test number in order collected. Numbers must be zero padded due to alphanumerical sorting. - warn_missing : dict - Whether to include relevant status badges for images and/or hashes. - Must have keys ``baseline_image`` and ``baseline_hash``. """ - def __init__(self, name, item, id, warn_missing): + def __init__(self, name, item, id): # Make the summary dictionary available as attributes self.__dict__ = item # Sort index for collection order self.id = id - # Whether to show image and/or hash status badges - self.warn_missing = warn_missing - # Name of test with module and test function together and separate self.full_name = name self.name = name.split('.')[-1] @@ -107,30 +103,6 @@ def __init__(self, name, item, id, warn_missing): ('hash', self.hash_status), ]] - @cached_property - def image_status(self): - """Status of the image comparison test.""" - if self.rms is None and self.tolerance is not None: - return 'match' - elif self.rms is not None: - return 'diff' - elif self.baseline_image is None: - return 'missing' - else: - raise ValueError('Unknown image result.') - - @cached_property - def hash_status(self): - """Status of the hash comparison test.""" - if self.baseline_hash is not None or self.result_hash is not None: - if self.baseline_hash is None: - return 'missing' - elif self.baseline_hash == self.result_hash: - return 'match' - else: - return 'diff' - return None - @cached_property def indexes(self): """Dictionary with strings optimized for sorting.""" @@ -178,8 +150,6 @@ def badges(self): """Additional badges to show beside overall status badge.""" for test_type, status_getter in [('image', image_status_msg), ('hash', hash_status_msg)]: status = getattr(self, f'{test_type}_status') - if not self.warn_missing[f'baseline_{test_type}']: - continue # not expected to exist if ( (status == 'missing') or (self.status == 'failed' and status == 'match') or @@ -198,6 +168,7 @@ def status_class(status): 'match': 'success', 'diff': 'danger', 'missing': 'warning', + 'generated': 'warning', } return classes[status] @@ -208,6 +179,7 @@ def image_status_msg(status): 'match': 'Baseline image matches', 'diff': 'Baseline image differs', 'missing': 'Baseline image not found', + 'generated': 'Baseline image was generated', } return messages[status] @@ -218,11 +190,12 @@ def hash_status_msg(status): 'match': 'Baseline hash matches', 'diff': 'Baseline hash differs', 'missing': 'Baseline hash not found', + 'generated': 'Baseline hash was generated', } return messages[status] -def generate_summary_html(results, results_dir): +def generate_summary_html(results, results_dir, hash_library=None): """Generate the HTML summary. Parameters @@ -231,6 +204,9 @@ def generate_summary_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -246,7 +222,7 @@ def generate_summary_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("base.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files for file in ['styles.css', 'extra.js', 'hash.svg', 'image.svg']: @@ -259,7 +235,7 @@ def generate_summary_html(results, results_dir): return html_file -def generate_summary_basic_html(results, results_dir): +def generate_summary_basic_html(results, results_dir, hash_library=None): """Generate the basic HTML summary. Parameters @@ -268,6 +244,9 @@ def generate_summary_basic_html(results, results_dir): The `pytest_mpl.plugin.ImageComparison._test_results` object. results_dir : Path Path to the output directory. + hash_library : str, optional, default=None + Filename of the generated hash library at the root of `results_dir`. + Will be linked to in HTML if not None. """ # Initialize Jinja @@ -278,7 +257,7 @@ def generate_summary_basic_html(results, results_dir): # Render HTML starting from the base template template = env.get_template("basic.html") - html = template.render(results=Results(results)) + html = template.render(results=Results(results), hash_library=hash_library) # Write files html_file = results_dir / 'fig_comparison_basic.html' diff --git a/pytest_mpl/summary/templates/base.html b/pytest_mpl/summary/templates/base.html index b34464b9..4ac3c66e 100644 --- a/pytest_mpl/summary/templates/base.html +++ b/pytest_mpl/summary/templates/base.html @@ -8,7 +8,7 @@ integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous"> <title>{{ results.title }} - + {% include 'navbar.html' %}
diff --git a/pytest_mpl/summary/templates/basic.html b/pytest_mpl/summary/templates/basic.html index 3feb1095..27e6417c 100644 --- a/pytest_mpl/summary/templates/basic.html +++ b/pytest_mpl/summary/templates/basic.html @@ -48,6 +48,11 @@

Image test comparison

of those have a matching baseline image {%- endif %}

+{% if hash_library -%} +

+ Download generated hash library +

+{%- endif %} diff --git a/pytest_mpl/summary/templates/extra.js b/pytest_mpl/summary/templates/extra.js index cb80eefe..d5c3c341 100644 --- a/pytest_mpl/summary/templates/extra.js +++ b/pytest_mpl/summary/templates/extra.js @@ -1,10 +1,3 @@ -// Remove all elements of class mpl-hash if hash test not run -if (document.body.classList[0] == 'no-hash-test') { - document.querySelectorAll('.mpl-hash').forEach(function (elem) { - elem.parentElement.removeChild(elem); - }); -} - // Enable tooltips var tooltipTriggerList = [].slice.call(document.querySelectorAll('[data-bs-toggle="tooltip"]')) var tooltipList = tooltipTriggerList.map(function (tooltipTriggerEl) { diff --git a/pytest_mpl/summary/templates/filter.html b/pytest_mpl/summary/templates/filter.html index 3cd8432f..81e282cc 100644 --- a/pytest_mpl/summary/templates/filter.html +++ b/pytest_mpl/summary/templates/filter.html @@ -16,7 +16,7 @@
Sort tests by...
{{ sort_option('status-sort', 'status', 'desc', default=true) }} {{ sort_option('collected-sort', 'collected', 'asc') }} {{ sort_option('test-name', 'name') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%} {{ sort_option('rms-sort', 'RMS', 'desc') }} {%- endif %} @@ -34,15 +34,15 @@
Show tests which have...
{{ filter_option('overall-failed', 'failed') }} {{ filter_option('overall-skipped', 'skipped') }} - {% if results.warn_missing['baseline_image'] -%} + {% if results.image_comparison -%}
{{ filter_option('image-match', 'matching images') }} {{ filter_option('image-diff', 'differing images') }} {{ filter_option('image-missing', 'no baseline image') }}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if results.hash_comparison -%} +
{{ filter_option('hash-match', 'matching hashes') }} {{ filter_option('hash-diff', 'differing hashes') }} {{ filter_option('hash-missing', 'no baseline hash') }} diff --git a/pytest_mpl/summary/templates/navbar.html b/pytest_mpl/summary/templates/navbar.html index 3ba762ac..17fa7bfd 100644 --- a/pytest_mpl/summary/templates/navbar.html +++ b/pytest_mpl/summary/templates/navbar.html @@ -9,6 +9,17 @@ + {% if hash_library -%} + + + + + + {%- endif %}
- {%- else -%} + {%- elif r.result_image -%} result image + {%- elif r.baseline_image -%} + baseline image {%- endif %} {% filter indent(width=8) -%} diff --git a/pytest_mpl/summary/templates/result_images.html b/pytest_mpl/summary/templates/result_images.html index 822bbd46..36ab2943 100644 --- a/pytest_mpl/summary/templates/result_images.html +++ b/pytest_mpl/summary/templates/result_images.html @@ -6,14 +6,14 @@
{{ r.module }}
{{ r.name }}
- {% if results.warn_missing['baseline_image'] -%} + {% if r.image_status and r.image_status != 'generated' -%}
{% macro image_card(file, name) -%}
{{ name }}
{% if file -%} - + {{ name }} {%- endif %}
@@ -41,21 +41,25 @@
{{ r.name }}
{%- endfilter %} {%- endmacro -%} - {% if results.warn_missing['baseline_image'] -%} -
+ {% if r.image_status -%} +
{{ r.image_status | image_status_msg }}
+ {% if r.image_status == 'match' or r.image_status == 'diff' -%}
{{ pre_data('rms', r.rms_str, 'RMS') }} {{ pre_data('tolerance', r.tolerance, 'Tolerance') }}
+ {%- endif %}
{%- endif %} - {% if results.warn_missing['baseline_hash'] -%} -
+ {% if r.hash_status -%} +
{{ r.hash_status | hash_status_msg }}
{{ pre_data('baseline_hash', r.baseline_hash, 'Baseline') }} + {% if r.hash_status != 'generated' -%} {{ pre_data('result_hash', r.result_hash, 'Result') }} + {%- endif %}
{%- endif %} diff --git a/pytest_mpl/summary/templates/styles.css b/pytest_mpl/summary/templates/styles.css index 3effb1df..c5870b09 100644 --- a/pytest_mpl/summary/templates/styles.css +++ b/pytest_mpl/summary/templates/styles.css @@ -1,6 +1,3 @@ -body.no-hash-test .mpl-hash { - display: none; -} .navbar-brand span.repo { font-weight: bold; } @@ -14,6 +11,9 @@ body.no-hash-test .mpl-hash { .nav-filtertools .nav-searchbar { flex: 10; } +.nav-filtertools svg { + vertical-align: text-bottom; +} #filterForm .spacer { flex: 1; } @@ -29,12 +29,12 @@ body.no-hash-test .mpl-hash { max-width: 400px; } } -.image-missing .mpl-image .card-body { - display: none; -} pre { white-space: pre-line; } +.tooltip-inner pre { + margin: 0; +} div.result div.status-badge button img { vertical-align: sub; } diff --git a/setup.cfg b/setup.cfg index cd193ab4..60478bd3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ classifiers = Topic :: Software Development :: Testing Topic :: Scientific/Engineering :: Visualization Programming Language :: Python - Programming Language :: Python :: 2 Programming Language :: Python :: 3 Operating System :: OS Independent License :: OSI Approved :: BSD License diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 0c688843..0f345f03 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -9,7 +9,8 @@ class MatchError(Exception): pass -def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None): +def diff_summary(baseline, result, baseline_hash_library=None, result_hash_library=None, + generating_hashes=False): """Diff a pytest-mpl summary dictionary. Parameters @@ -26,6 +27,9 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra Path to the "baseline" image hash library. Result hashes in the baseline summary are updated to these values to handle different Matplotlib versions. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ if baseline_hash_library and baseline_hash_library.exists(): # Load "correct" baseline hashes @@ -57,10 +61,13 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Swap the baseline and result hashes in the summary # for the corresponding hashes in each hash library - if baseline_hash_library and test in baseline_hash_library: + if baseline_hash_library and test in baseline_hash_library and not generating_hashes: baseline_summary = replace_hash(baseline_summary, 'baseline_hash', baseline_hash_library[test]) if result_hash_library: + if generating_hashes: # Newly generate result will appear as baseline_hash + baseline_summary = replace_hash(baseline_summary, 'baseline_hash', + result_hash_library[test]) baseline_summary = replace_hash(baseline_summary, 'result_hash', result_hash_library[test]) diff --git a/tests/subtests/summaries/test_default.json b/tests/subtests/summaries/test_default.json index a4acadc6..b4b51444 100644 --- a/tests/subtests/summaries/test_default.json +++ b/tests/subtests/summaries/test_default.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": null, "diff_image": null, diff --git a/tests/subtests/summaries/test_generate.json b/tests/subtests/summaries/test_generate.json new file mode 100644 index 00000000..6f9f78e2 --- /dev/null +++ b/tests/subtests/summaries/test_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_hashes_only.json b/tests/subtests/summaries/test_generate_hashes_only.json new file mode 100644 index 00000000..8dee25fa --- /dev/null +++ b/tests/subtests/summaries/test_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_generate_images_only.json b/tests/subtests/summaries/test_generate_images_only.json new file mode 100644 index 00000000..7720807d --- /dev/null +++ b/tests/subtests/summaries/test_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_hash.json b/tests/subtests/summaries/test_hash.json index bec04c5a..4304bf03 100644 --- a/tests/subtests/summaries/test_hash.json +++ b/tests/subtests/summaries/test_hash.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_generate.json b/tests/subtests/summaries/test_html_generate.json new file mode 100644 index 00000000..84a8e8b9 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": "generated", + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_hashes_only.json b/tests/subtests/summaries/test_html_generate_hashes_only.json new file mode 100644 index 00000000..0a813734 --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_hashes_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_imatch/result.png", + "baseline_hash": "4a47c9b7920779cc83eabe2bbb64b9c40745d9d8abfa57857f93a5d8f12a5a03", + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", + "rms": 11.100353848213828, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmatch_idiff/result.png", + "baseline_hash": "2b48790b0a2cee4b41cdb9820336acaf229ba811ae21c6a92b4b92838843adfa", + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmatch_imissing/result.png", + "baseline_hash": "e937fa1997d088c904ca35b1ab542e2285ea47b84df976490380f9c5f5b5f8ae", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch/result.png", + "baseline_hash": "2cae8f315d44b06de8f45d937af46a67bd1389edd6e4cde32f9feb4b7472284f", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", + "rms": 11.182677079602481, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_idiff/result.png", + "baseline_hash": "927521206ef454a25417e3ba0bd3235c84518cb202c2d1fa7afcfdfcde5fdcde", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hdiff_imissing/result.png", + "baseline_hash": "afc411cfa34db3a5819ac4127704e86acf27d24d1ea2410718853d3d7e1d6ae0", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_imatch/result.png", + "baseline_hash": "7ee8370efdc4b767634d12355657ca4f2460176670c07b31f3fb72cea0e79856", + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", + "rms": 12.12938597648977, + "tolerance": 2, + "result_image": "subtests.subtest.test_hmissing_idiff/result.png", + "baseline_hash": "4eeda1d349f4b0f26df97df41ba5410dce2b1c7ed520062d58f3c5f0e3790ebd", + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "failed", + "image_status": "missing", + "hash_status": "generated", + "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "baseline_image": null, + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": "subtests.subtest.test_hmissing_imissing/result.png", + "baseline_hash": "5101e60ac100cf2c2f418a0a6a382aae0060339e76718730344f539b61f7dc7e", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 200, + "result_image": "subtests.subtest.test_hdiff_imatch_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "failed", + "image_status": "diff", + "hash_status": "generated", + "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", + "rms": 29.260332173249314, + "tolerance": 3, + "result_image": "subtests.subtest.test_hdiff_idiff_tolerance/result.png", + "baseline_hash": "510b3273d63a2a26a27e788ff0f090e86c9df7f9f191b7c566321c57de8266d6", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_savefig/result.png", + "baseline_hash": "8864803a4b4026d8c6dc0ab950228793ea255cd9b6c629c39db9e6315a9af6bc", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_style/result.png", + "baseline_hash": "e3c8de36c2bad7dca131e4cbbfe229f882b5beec62750fb7da29314fd6a1ff13", + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "passed", + "image_status": "match", + "hash_status": "generated", + "status_msg": "Image comparison passed.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": 2, + "result_image": "subtests.subtest.test_hdiff_imatch_removetext/result.png", + "baseline_hash": "e4c06cf613c6836c1b1202abaae69cf65bc2232a8e31ab1040454bedc8e31e7a", + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_generate_images_only.json b/tests/subtests/summaries/test_html_generate_images_only.json new file mode 100644 index 00000000..bed75fae --- /dev/null +++ b/tests/subtests/summaries/test_html_generate_images_only.json @@ -0,0 +1,184 @@ +{ + "subtests.subtest.test_hmatch_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmatch_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmatch_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imatch": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_idiff": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hmissing_imissing": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hmissing_imissing/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_idiff_tolerance": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_savefig": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_style": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + }, + "subtests.subtest.test_hdiff_imatch_removetext": { + "status": "skipped", + "image_status": "generated", + "hash_status": null, + "status_msg": "Skipped test, since generating image.", + "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", + "diff_image": null, + "rms": null, + "tolerance": null, + "result_image": null, + "baseline_hash": null, + "result_hash": null + } +} diff --git a/tests/subtests/summaries/test_html_hashes_only.json b/tests/subtests/summaries/test_html_hashes_only.json index 448ce9c2..a55666ac 100644 --- a/tests/subtests/summaries/test_html_hashes_only.json +++ b/tests/subtests/summaries/test_html_hashes_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.", + "image_status": null, + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.", + "image_status": null, + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_html_images_only.json b/tests/subtests/summaries/test_html_images_only.json index c8a5058c..0b33035e 100644 --- a/tests/subtests/summaries/test_html_images_only.json +++ b/tests/subtests/summaries/test_html_images_only.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -34,6 +40,8 @@ }, "subtests.subtest.test_hdiff_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, @@ -45,6 +53,8 @@ }, "subtests.subtest.test_hdiff_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", @@ -56,6 +66,8 @@ }, "subtests.subtest.test_hdiff_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -67,6 +79,8 @@ }, "subtests.subtest.test_hmissing_imatch": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, @@ -78,6 +92,8 @@ }, "subtests.subtest.test_hmissing_idiff": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", @@ -89,6 +105,8 @@ }, "subtests.subtest.test_hmissing_imissing": { "status": "failed", + "image_status": "missing", + "hash_status": null, "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -100,6 +118,8 @@ }, "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, @@ -111,6 +131,8 @@ }, "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", + "image_status": "diff", + "hash_status": null, "status_msg": "REGEX:Error: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", @@ -122,6 +144,8 @@ }, "subtests.subtest.test_hdiff_imatch_savefig": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, @@ -133,6 +157,8 @@ }, "subtests.subtest.test_hdiff_imatch_style": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, @@ -144,6 +170,8 @@ }, "subtests.subtest.test_hdiff_imatch_removetext": { "status": "passed", + "image_status": "match", + "hash_status": null, "status_msg": "Image comparison passed.", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, diff --git a/tests/subtests/summaries/test_hybrid.json b/tests/subtests/summaries/test_hybrid.json index 2f2930b0..851e67f1 100644 --- a/tests/subtests/summaries/test_hybrid.json +++ b/tests/subtests/summaries/test_hybrid.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": null, + "hash_status": "match", "status_msg": "Test hash matches baseline hash.", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/summaries/test_results_always.json b/tests/subtests/summaries/test_results_always.json index c42fee29..ea7af1d2 100644 --- a/tests/subtests/summaries/test_results_always.json +++ b/tests/subtests/summaries/test_results_always.json @@ -1,6 +1,8 @@ { "subtests.subtest.test_hmatch_imatch": { "status": "passed", + "image_status": "match", + "hash_status": "match", "status_msg": "Test hash matches baseline hash.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded.", "baseline_image": "subtests.subtest.test_hmatch_imatch/baseline.png", "diff_image": null, @@ -12,6 +14,8 @@ }, "subtests.subtest.test_hmatch_idiff": { "status": "passed", + "image_status": "diff", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png", @@ -23,6 +27,8 @@ }, "subtests.subtest.test_hmatch_imissing": { "status": "passed", + "image_status": "missing", + "hash_status": "match", "status_msg": "REGEX:Test hash matches baseline hash\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", "baseline_image": null, "diff_image": null, @@ -35,6 +41,8 @@ "subtests.subtest.test_hdiff_imatch": { "status": "failed", "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch/baseline.png", "diff_image": null, "rms": null, @@ -46,6 +54,8 @@ "subtests.subtest.test_hdiff_idiff": { "status": "failed", "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 11\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png", "rms": 11.182677079602481, @@ -57,6 +67,8 @@ "subtests.subtest.test_hdiff_imissing": { "status": "failed", "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "diff", "baseline_image": null, "diff_image": null, "rms": null, @@ -68,6 +80,8 @@ "subtests.subtest.test_hmissing_imatch": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_imatch/baseline.png", "diff_image": null, "rms": null, @@ -79,6 +93,8 @@ "subtests.subtest.test_hmissing_idiff": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 12\\.1[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 2", + "image_status": "diff", + "hash_status": "missing", "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png", "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png", "rms": 12.12938597648977, @@ -90,6 +106,8 @@ "subtests.subtest.test_hmissing_imissing": { "status": "failed", "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.\n\nImage comparison test\n---------------------\nImage file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png", + "image_status": "missing", + "hash_status": "missing", "baseline_image": null, "diff_image": null, "rms": null, @@ -101,6 +119,8 @@ "subtests.subtest.test_hdiff_imatch_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_tolerance/baseline.png", "diff_image": null, "rms": null, @@ -112,6 +132,8 @@ "subtests.subtest.test_hdiff_idiff_tolerance": { "status": "failed", "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.\n\nImage comparison test\n---------------------\nError: Image files did not match\\.\n RMS Value: 29\\.2[0-9]*\n Expected: \n .*baseline\\.png\n Actual: \n .*result\\.png\n Difference:\n .*result-failed-diff\\.png\n Tolerance: \n 3", + "image_status": "diff", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png", "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png", "rms": 29.260332173249314, @@ -123,6 +145,8 @@ "subtests.subtest.test_hdiff_imatch_savefig": { "status": "failed", "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_savefig/baseline.png", "diff_image": null, "rms": null, @@ -134,6 +158,8 @@ "subtests.subtest.test_hdiff_imatch_style": { "status": "failed", "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_style/baseline.png", "diff_image": null, "rms": null, @@ -145,6 +171,8 @@ "subtests.subtest.test_hdiff_imatch_removetext": { "status": "failed", "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.\n\nImage comparison test\n---------------------\nThe comparison to the baseline image succeeded\\.", + "image_status": "match", + "hash_status": "diff", "baseline_image": "subtests.subtest.test_hdiff_imatch_removetext/baseline.png", "diff_image": null, "rms": null, diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 48a2f461..eb85aa5c 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -34,6 +34,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, + has_result_hashes=False, generating_hashes=False, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -49,6 +50,12 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru Summaries to generate in addition to `json`. xfail : bool, optional, default=True Whether the overall pytest run should fail. + has_result_hashes : bool or str, optional, default=False + Whether a hash library is expected to exist in the results directory. + If a string, this is the name of the expected results file. + generating_hashes : bool, optional, default=False + Whether `--mpl-generate-hash-library` was specified and + both of `--mpl-hash-library` and `hash_library=` were not. """ # Parse arguments if summaries is None: @@ -105,11 +112,30 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru # Compare summaries diff_summary(baseline_summary, result_summary, - baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY) + baseline_hash_library=HASH_LIBRARY, result_hash_library=RESULT_LIBRARY, + generating_hashes=generating_hashes) # Ensure reported images exist assert_existence(result_summary, path=results_path) + # Get expected name for the hash library saved to the results directory + if isinstance(has_result_hashes, str): + result_hash_file = tmp_path / 'results' / has_result_hashes + has_result_hashes = True # convert to bool after processing str + else: + result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name + + # Compare the generated hash library to the expected hash library + if has_result_hashes: + assert result_hash_file.exists() + with open(RESULT_LIBRARY, "r") as f: + baseline = json.load(f) + with open(result_hash_file, "r") as f: + result = json.load(f) + diff_summary({'a': baseline}, {'a': result}) + else: + assert not result_hash_file.exists() + def test_default(tmp_path): run_subtest('test_default', tmp_path, []) @@ -128,13 +154,15 @@ def test_hybrid(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_results_always(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'], + has_result_hashes=True) @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html']) + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -142,7 +170,8 @@ def test_html(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_html_hashes_only(tmp_path): - run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html']) + run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison.html').exists() assert (tmp_path / 'results' / 'extra.js').exists() assert (tmp_path / 'results' / 'styles.css').exists() @@ -158,5 +187,67 @@ def test_html_images_only(tmp_path): @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") def test_basic_html(tmp_path): run_subtest('test_results_always', tmp_path, - [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html']) + [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'], + has_result_hashes=True) assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + xfail=False, generating_hashes=True) + + +def test_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], xfail=False) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + generating_hashes=True) + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate(tmp_path): + # generating hashes and images; no testing + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +def test_html_generate_images_only(tmp_path): + # generating images; no testing + run_subtest('test_html_generate_images_only', tmp_path, + [rf'--mpl-generate-path={tmp_path}'], + summaries=['html'], xfail=False) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_generate_hashes_only(tmp_path): + # generating hashes; testing images + run_subtest('test_html_generate_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], has_result_hashes="test_hashes.json", generating_hashes=True) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + + +@pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version") +def test_html_run_generate_hashes_only(tmp_path): + # generating hashes; testing hashes + run_subtest('test_html_hashes_only', tmp_path, + [rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}', + HASH_LIBRARY_FLAG], + summaries=['html'], has_result_hashes="test_hashes.json") + assert (tmp_path / 'results' / 'fig_comparison.html').exists() pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy