-
Notifications
You must be signed in to change notification settings - Fork 5
Expand file tree
/
Copy pathcompare.py
More file actions
45 lines (40 loc) · 1.59 KB
/
compare.py
File metadata and controls
45 lines (40 loc) · 1.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
import numpy as np
import torch
PARTS = {
"baddbmm": [
"sliced_alibi",
"query_layer",
"key_layer",
"value_layer",
"beta",
"alpha",
"matmul_result",
],
"softmax": ["attention_scores", "attention_mask", "attn_weights", "attention_probs"],
"bmm": ["value_layer", "attention_probs_reshaped", "context_layer", "dense", "residual", "dropout"],
"mlp": ["init", "gelu", "output"],
}
AA = []
BB = []
for gen_step in range(20):
for layer in range(70):
for part_name, parts in PARTS.items():
for name in parts:
try:
A = np.load(f"tensors/{gen_step}_python_{part_name}_{name}_{layer}.npy")
B = np.load(f"tensors/{gen_step}_rust_{part_name}_{name}_{layer}.npy")
except Exception:
continue
print("gen_step", gen_step, "part", part_name, "Name ", name, "layer", layer, np.allclose(A, B, rtol=1e-8, atol=1e-10) and A.shape == B.shape)
if gen_step == 0 and part_name == "baddbmm":
AA.append(A)
BB.append(B)
if A.shape != B.shape:
import ipdb;ipdb.set_trace()
if not np.allclose(A, B, rtol=1e-8, atol=1e-10):
import ipdb;ipdb.set_trace()
# torch.testing.assert_close(torch.from_numpy(A), torch.from_numpy(B))
try:
torch.testing.assert_close(torch.from_numpy(A), torch.from_numpy(B))
except Exception as e:
print(e)