Skip to content

test: refactor tests using pytest.mark.parametrize #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 40 commits into from
Aug 24, 2022
Merged

test: refactor tests using pytest.mark.parametrize #55

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
dfb1b63
test: use `torch.allclose` to compare tensors
XuehaiPan Aug 8, 2022
63d5012
test: refactor tests using `pytest.mark.parametrize`
XuehaiPan Aug 8, 2022
d2ea7fc
chore: update test ids
XuehaiPan Aug 8, 2022
29284c4
chore: update test ids
XuehaiPan Aug 8, 2022
591999f
chore(workflows): increase timeout
XuehaiPan Aug 8, 2022
803c7b9
test: reorganize tests
XuehaiPan Aug 8, 2022
99a561d
feat: parallel testing
XuehaiPan Aug 8, 2022
5b7dedb
test: reorganize tests
XuehaiPan Aug 8, 2022
4992712
test: loop for more update iterations
XuehaiPan Aug 8, 2022
85bddb0
test: reduce number of tests
XuehaiPan Aug 8, 2022
92fa5c7
test: reduce number of tests
XuehaiPan Aug 8, 2022
acdb41a
test: update assert
XuehaiPan Aug 8, 2022
cf45c8e
test: rename variable
XuehaiPan Aug 9, 2022
aba2efc
test: update assert
XuehaiPan Aug 9, 2022
2d28f49
docs(CHANGELOG): update CHANGELOG.md
XuehaiPan Aug 10, 2022
2745eb1
test: cov project name
XuehaiPan Aug 10, 2022
de69946
test: update tests
XuehaiPan Aug 11, 2022
468c4dc
test: set `CUBLAS_WORKSPACE_CONFIG`
XuehaiPan Aug 12, 2022
c7e72c5
test: compare diffs
XuehaiPan Aug 15, 2022
83717bb
test: update tol
XuehaiPan Aug 15, 2022
998b400
test: test float32
XuehaiPan Aug 15, 2022
933eb5d
test: show tol in assert messages
XuehaiPan Aug 15, 2022
876e9d3
test: use smaller network for tests
XuehaiPan Aug 16, 2022
9450669
Merge remote-tracking branch 'upstream/main' into fix-tests
Benjamin-eecs Aug 22, 2022
7034ec3
fix: change the order of add and multiply in sgd
Benjamin-eecs Aug 22, 2022
4b30fc0
Merge branch 'main' into fix-tests
XuehaiPan Aug 22, 2022
1b0a35c
fix: correct test writing and pass sgd
Benjamin-eecs Aug 23, 2022
2a09c8c
Merge branch 'fix-tests' of https://github.com/XuehaiPan/TorchOpt int…
Benjamin-eecs Aug 23, 2022
d6179f6
fix: correct test writing and fix other optims
Benjamin-eecs Aug 23, 2022
48f4a94
to(tests): high level non differentiable optimizer unfixed
Benjamin-eecs Aug 23, 2022
4ddf147
to(tests): high level non differentiable optimizer unfixed
Benjamin-eecs Aug 23, 2022
d77cdd8
to(tests): high level non differentiable optimizer unfixed
Benjamin-eecs Aug 23, 2022
f32ef9f
test: disable parallel testing
XuehaiPan Aug 24, 2022
629a36f
fix(transform): fix momentum trace
XuehaiPan Aug 24, 2022
ca580a6
refactor: refactor transform and utils
XuehaiPan Aug 24, 2022
6771907
test: test inplace operators
XuehaiPan Aug 24, 2022
579e786
fix: fix RMSProp optimizer
XuehaiPan Aug 24, 2022
0cb3c44
fix: fix Makefile
XuehaiPan Aug 24, 2022
51d15b3
lint: appease linters
XuehaiPan Aug 24, 2022
1e1d86f
chore: update unused function
XuehaiPan Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
test: reorganize tests
  • Loading branch information
@XuehaiPan
XuehaiPan committed Aug 14, 2022
commit 803c7b9594548f6e55cdf04205210a1ef7de2d40
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ addlicense-install: go-install

pytest: pytest-install
cd tests && \
$(PYTHON) -m pytest unit --verbose --color=yes --durations=0 \
$(PYTHON) -m pytest . --verbose --color=yes --durations=0 \
--cov="$(PROJECT_PATH)" --cov-report=xml --cov-report=term-missing

test: pytest
Expand Down
72 changes: 72 additions & 0 deletions tests/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
# Copyright 2022 MetaOPT Team. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import copy
import itertools
import os
import random
from typing import Optional, Tuple, Union

import numpy as np
import pytest
import torch
import torch.nn as nn
from torch.utils import data
from torchvision import models


def parametrize(**argvalues) -> pytest.mark.parametrize:
arguments = tuple(argvalues)
argvalues = tuple(itertools.product(*tuple(map(argvalues.get, arguments))))
ids = tuple(
'-'.join(f'{arg}({val})' for arg, val in zip(arguments, values)) for values in argvalues
)

return pytest.mark.parametrize(arguments, argvalues, ids=ids)


def seed_everything(seed: int) -> None:
os.environ['PYTHONHASHSEED'] = str(seed)

random.seed(seed)
np.random.seed(seed)

torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
try:
torch.use_deterministic_algorithms(True)
except AttributeError:
pass


def get_models(
device: Optional[Union[str, torch.device]] = None, dtype: torch.dtype = torch.float32
) -> Tuple[nn.Module, nn.Module, data.DataLoader]:
seed_everything(seed=42)

model = models.resnet18().to(dtype=dtype)
model_ref = copy.deepcopy(model)
if device is not None:
model = model.to(device=torch.device(device))
model_ref = model_ref.to(device=torch.device(device))

batch_size = 8
dataset = data.TensorDataset(
torch.randn(batch_size * 2, 3, 224, 224), torch.randint(0, 1000, (batch_size * 2,))
)
loader = data.DataLoader(dataset, batch_size, shuffle=False)

return model, model_ref, loader
63 changes: 12 additions & 51 deletions ...nit/high_level/test_high_level_inplace.py → tests/high_level/test_high_level_inplace.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,63 +13,24 @@
# limitations under the License.
# ==============================================================================

import copy
import itertools
import random
from typing import Optional, Tuple, Union
from typing import Tuple

import numpy as np
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
from torchvision import models

import helpers
import torchopt


def parametrize(**argvalues) -> pytest.mark.parametrize:
arguments = tuple(argvalues)
argvalues = tuple(itertools.product(*tuple(map(argvalues.get, arguments))))
ids = tuple(
'-'.join(f'{arg}({val})' for arg, val in zip(arguments, values)) for values in argvalues
)

return pytest.mark.parametrize(arguments, argvalues, ids=ids)


def get_models(
device: Optional[Union[str, torch.device]] = None, dtype: torch.dtype = torch.float32
) -> Tuple[nn.Module, nn.Module, data.DataLoader]:
random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)

model = models.resnet18().to(dtype=dtype)
model_ref = copy.deepcopy(model)
if device is not None:
model = model.to(device=torch.device(device))
model_ref = model_ref.to(device=torch.device(device))

batch_size = 8
dataset = data.TensorDataset(
torch.randn(batch_size * 2, 3, 224, 224), torch.randint(0, 1000, (batch_size * 2,))
)
loader = data.DataLoader(dataset, batch_size, shuffle=False)

return model, model_ref, loader


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
momentum=[0.0, 0.1, 0.2],
nesterov=[False, True],
)
def test_sgd(dtype: torch.dtype, lr: float, momentum: float, nesterov: bool) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

optim = torchopt.SGD(
model.parameters(), lr, momentum=(momentum if momentum != 0.0 else None), nesterov=nesterov
Expand Down Expand Up @@ -102,14 +63,14 @@ def test_sgd(dtype: torch.dtype, lr: float, momentum: float, nesterov: bool) ->
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
eps=[1e-8, 1e-6],
)
def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

optim = torchopt.Adam(model.parameters(), lr, b1=betas[0], b2=betas[1], eps=eps, eps_root=0.0)
optim_ref = torch.optim.Adam(
Expand Down Expand Up @@ -140,7 +101,7 @@ def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: fl
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
Expand All @@ -149,7 +110,7 @@ def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: fl
def test_accelerated_adam_cpu(
dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float
) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

optim = torchopt.Adam(
model.parameters(),
Expand Down Expand Up @@ -189,7 +150,7 @@ def test_accelerated_adam_cpu(


@pytest.mark.skipif(not torch.cuda.is_available(), reason='No CUDA device available.')
@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
Expand All @@ -199,7 +160,7 @@ def test_accelerated_adam_cuda(
dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float
) -> None:
device = 'cuda'
model, model_ref, loader = get_models(device=device, dtype=dtype)
model, model_ref, loader = helpers.get_models(device=device, dtype=dtype)

optim = torchopt.Adam(
model.parameters(),
Expand Down Expand Up @@ -239,7 +200,7 @@ def test_accelerated_adam_cuda(
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
alpha=[0.9, 0.99],
Expand All @@ -250,7 +211,7 @@ def test_accelerated_adam_cuda(
def test_rmsprop(
dtype: torch.dtype, lr: float, alpha: float, eps: float, momentum: float, centered: bool
) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

optim = torchopt.RMSProp(
model.parameters(),
Expand Down
63 changes: 12 additions & 51 deletions .../unit/low_level/test_low_level_inplace.py → tests/low_level/test_low_level_inplace.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,64 +13,25 @@
# limitations under the License.
# ==============================================================================

import copy
import itertools
import random
from typing import Optional, Tuple, Union
from typing import Tuple

import functorch
import numpy as np
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils import data
from torchvision import models

import helpers
import torchopt


def parametrize(**argvalues) -> pytest.mark.parametrize:
arguments = tuple(argvalues)
argvalues = tuple(itertools.product(*tuple(map(argvalues.get, arguments))))
ids = tuple(
'-'.join(f'{arg}({val})' for arg, val in zip(arguments, values)) for values in argvalues
)

return pytest.mark.parametrize(arguments, argvalues, ids=ids)


def get_models(
device: Optional[Union[str, torch.device]] = None, dtype: torch.dtype = torch.float32
) -> Tuple[nn.Module, nn.Module, data.DataLoader]:
random.seed(0)
np.random.seed(0)
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)

model = models.resnet18().to(dtype=dtype)
model_ref = copy.deepcopy(model)
if device is not None:
model = model.to(device=torch.device(device))
model_ref = model_ref.to(device=torch.device(device))

batch_size = 8
dataset = data.TensorDataset(
torch.randn(batch_size * 2, 3, 224, 224), torch.randint(0, 1000, (batch_size * 2,))
)
loader = data.DataLoader(dataset, batch_size, shuffle=False)

return model, model_ref, loader


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
momentum=[0.0, 0.1, 0.2],
nesterov=[False, True],
)
def test_sgd(dtype: torch.dtype, lr: float, momentum: float, nesterov: bool) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

fun, params, buffers = functorch.make_functional_with_buffers(model)
optim = torchopt.sgd(lr, momentum=(momentum if momentum != 0.0 else None), nesterov=nesterov)
Expand Down Expand Up @@ -103,14 +64,14 @@ def test_sgd(dtype: torch.dtype, lr: float, momentum: float, nesterov: bool) ->
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
eps=[1e-8, 1e-6],
)
def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

fun, params, buffers = functorch.make_functional_with_buffers(model)
optim = torchopt.adam(lr, b1=betas[0], b2=betas[1], eps=eps, eps_root=0.0)
Expand Down Expand Up @@ -143,7 +104,7 @@ def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: fl
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
Expand All @@ -152,7 +113,7 @@ def test_adam(dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: fl
def test_accelerated_adam_cpu(
dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float
) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

fun, params, buffers = functorch.make_functional_with_buffers(model)
optim = torchopt.adam(
Expand Down Expand Up @@ -188,7 +149,7 @@ def test_accelerated_adam_cpu(


@pytest.mark.skipif(not torch.cuda.is_available(), reason='No CUDA device available.')
@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
betas=[(0.9, 0.999), (0.95, 0.9995)],
Expand All @@ -198,7 +159,7 @@ def test_accelerated_adam_cuda(
dtype: torch.dtype, lr: float, betas: Tuple[float, float], eps: float
) -> None:
device = 'cuda'
model, model_ref, loader = get_models(device=device, dtype=dtype)
model, model_ref, loader = helpers.get_models(device=device, dtype=dtype)

fun, params, buffers = functorch.make_functional_with_buffers(model)
optim = torchopt.adam(
Expand Down Expand Up @@ -234,7 +195,7 @@ def test_accelerated_adam_cuda(
assert torch.allclose(b, b_ref), f'{b!r} != {b_ref!r}'


@parametrize(
@helpers.parametrize(
dtype=[torch.float32, torch.float64],
lr=[1e-3, 1e-4, 1e-5],
alpha=[0.9, 0.99],
Expand All @@ -245,7 +206,7 @@ def test_accelerated_adam_cuda(
def test_rmsprop(
dtype: torch.dtype, lr: float, alpha: float, eps: float, momentum: float, centered: bool
) -> None:
model, model_ref, loader = get_models(device='cpu', dtype=dtype)
model, model_ref, loader = helpers.get_models(device='cpu', dtype=dtype)

fun, params, buffers = functorch.make_functional_with_buffers(model)
optim = torchopt.rmsprop(
Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy