-
Notifications
You must be signed in to change notification settings - Fork 28
/
eval.py
118 lines (95 loc) · 4.24 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import torch
import torch.nn.functional as F
import numpy as np
from sklearn.metrics import roc_auc_score, f1_score
def eval_f1(y_true, y_pred):
acc_list = []
y_true = y_true.detach().cpu().numpy()
y_pred = y_pred.argmax(dim=-1, keepdim=True).detach().cpu().numpy()
for i in range(y_true.shape[1]):
f1 = f1_score(y_true, y_pred, average='micro')
acc_list.append(f1)
return sum(acc_list) / len(acc_list)
def eval_acc(y_true, y_pred):
acc_list = []
y_true = y_true.detach().cpu().numpy()
y_pred = y_pred.argmax(dim=-1, keepdim=True).detach().cpu().numpy()
for i in range(y_true.shape[1]):
is_labeled = y_true[:, i] == y_true[:, i]
correct = y_true[is_labeled, i] == y_pred[is_labeled, i]
acc_list.append(float(np.sum(correct)) / len(correct))
return sum(acc_list) / len(acc_list)
def eval_rocauc(y_true, y_pred):
""" adapted from ogb
https://github.com/snap-stanford/ogb/blob/master/ogb/nodeproppred/evaluate.py"""
rocauc_list = []
y_true = y_true.detach().cpu().numpy()
if y_true.shape[1] == 1:
# use the predicted class for single-class classification
y_pred = F.softmax(y_pred, dim=-1)[:, 1].unsqueeze(1).cpu().numpy()
else:
y_pred = y_pred.detach().cpu().numpy()
for i in range(y_true.shape[1]):
# AUC is only defined when there is at least one positive data.
if np.sum(y_true[:, i] == 1) > 0 and np.sum(y_true[:, i] == 0) > 0:
is_labeled = y_true[:, i] == y_true[:, i]
score = roc_auc_score(y_true[is_labeled, i], y_pred[is_labeled, i])
rocauc_list.append(score)
if len(rocauc_list) == 0:
raise RuntimeError(
'No positively labeled data available. Cannot compute ROC-AUC.')
return sum(rocauc_list) / len(rocauc_list)
@torch.no_grad()
def evaluate(model, dataset, split_idx, eval_func, criterion, args):
model.eval()
if args.method == 'nodeformer':
out, _ = model(dataset.graph['node_feat'], dataset.graph['adjs'], args.tau)
else:
out = model(dataset)
train_acc = eval_func(
dataset.label[split_idx['train']], out[split_idx['train']])
valid_acc = eval_func(
dataset.label[split_idx['valid']], out[split_idx['valid']])
test_acc = eval_func(
dataset.label[split_idx['test']], out[split_idx['test']])
if args.dataset in ('yelp-chi', 'deezer-europe', 'twitch-e', 'fb100', 'ogbn-proteins'):
if dataset.label.shape[1] == 1:
true_label = F.one_hot(dataset.label, dataset.label.max() + 1).squeeze(1)
else:
true_label = dataset.label
valid_loss = criterion(out[split_idx['valid']], true_label.squeeze(1)[
split_idx['valid']].to(torch.float))
else:
out = F.log_softmax(out, dim=1)
valid_loss = criterion(
out[split_idx['valid']], dataset.label.squeeze(1)[split_idx['valid']])
return train_acc, valid_acc, test_acc, valid_loss, out
@torch.no_grad()
def evaluate_cpu(model, dataset, split_idx, eval_func, criterion, args, result=None):
model.eval()
model.to(torch.device("cpu"))
dataset.label = dataset.label.to(torch.device("cpu"))
adjs_, x = dataset.graph['adjs'], dataset.graph['node_feat']
adjs = []
adjs.append(adjs_[0])
for k in range(args.rb_order - 1):
adjs.append(adjs_[k + 1])
out, _ = model(x, adjs)
train_acc = eval_func(
dataset.label[split_idx['train']], out[split_idx['train']])
valid_acc = eval_func(
dataset.label[split_idx['valid']], out[split_idx['valid']])
test_acc = eval_func(
dataset.label[split_idx['test']], out[split_idx['test']])
if args.dataset in ('yelp-chi', 'deezer-europe', 'twitch-e', 'fb100', 'ogbn-proteins'):
if dataset.label.shape[1] == 1:
true_label = F.one_hot(dataset.label, dataset.label.max() + 1).squeeze(1)
else:
true_label = dataset.label
valid_loss = criterion(out[split_idx['valid']], true_label.squeeze(1)[
split_idx['valid']].to(torch.float))
else:
out = F.log_softmax(out, dim=1)
valid_loss = criterion(
out[split_idx['valid']], dataset.label.squeeze(1)[split_idx['valid']])
return train_acc, valid_acc, test_acc, valid_loss, out