Evaluation
Protocols and Metrics Code Examples
"Evaluation Pluging" Example
1
# --- CONFIG
2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
3
# ---------
4
โ€‹
5
# --- TRANSFORMATIONS
6
train_transform = transforms.Compose([
7
RandomCrop(28, padding=4),
8
ToTensor(),
9
transforms.Normalize((0.1307,), (0.3081,))
10
])
11
test_transform = transforms.Compose([
12
ToTensor(),
13
transforms.Normalize((0.1307,), (0.3081,))
14
])
15
# ---------
16
โ€‹
17
# --- BENCHMARK CREATION
18
mnist_train = MNIST('./data/mnist', train=True,
19
download=True, transform=train_transform)
20
mnist_test = MNIST('./data/mnist', train=False,
21
download=True, transform=test_transform)
22
benchmark = nc_benchmark(
23
mnist_train, mnist_test, 5, task_labels=False, seed=1234)
24
# ---------
25
โ€‹
26
# MODEL CREATION
27
model = SimpleMLP(num_classes=benchmark.n_classes)
28
โ€‹
29
# DEFINE THE EVALUATION PLUGIN AND LOGGER
30
# The evaluation plugin manages the metrics computation.
31
# It takes as argument a list of metrics and a list of loggers.
32
# The evaluation plugin calls the loggers to serialize the metrics
33
# and save them in persistent memory or print them in the standard output.
34
โ€‹
35
# log to Tensorboard
36
tb_logger = TensorboardLogger()
37
โ€‹
38
# log to text file
39
text_logger = TextLogger(open('log.txt', 'a'))
40
โ€‹
41
# print to stdout
42
interactive_logger = InteractiveLogger()
43
โ€‹
44
eval_plugin = EvaluationPlugin(
45
accuracy_metrics(minibatch=True, epoch=True, experience=True, stream=True),
46
loss_metrics(minibatch=True, epoch=True, experience=True, stream=True),
47
timing_metrics(epoch=True, epoch_running=True),
48
cpu_usage_metrics(experience=True),
49
ExperienceForgetting(),
50
StreamConfusionMatrix(num_classes=benchmark.n_classes, save_image=False),
51
disk_usage_metrics(minibatch=True, epoch=True, experience=True, stream=True),
52
loggers=[interactive_logger, text_logger, tb_logger])
53
โ€‹
54
# CREATE THE STRATEGY INSTANCE (NAIVE)
55
cl_strategy = Naive(
56
model, SGD(model.parameters(), lr=0.001, momentum=0.9),
57
CrossEntropyLoss(), train_mb_size=500, train_epochs=1, test_mb_size=100,
58
device=device, evaluator=eval_plugin)
59
โ€‹
60
# TRAINING LOOP
61
print('Starting experiment...')
62
results = []
63
for experience in benchmark.train_stream:
64
print("Start of experience: ", experience.current_experience)
65
print("Current Classes: ", experience.classes_in_this_experience)
66
โ€‹
67
# train returns a dictionary which contains all the metric values
68
res = cl_strategy.train(experience, num_workers=4)
69
print('Training completed')
70
โ€‹
71
print('Computing accuracy on the whole test set')
72
# test also returns a dictionary which contains all the metric values
73
results.append(cl_strategy.eval(benchmark.test_stream, num_workers=4))
Copied!

๐Ÿค Run it on Google Colab

You can run this chapter and play with it on Google Colaboratory:
Notebook currently unavailable.
Last modified 4mo ago
Export as PDF
Copy link