-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathoptimizers.py
More file actions
147 lines (119 loc) · 5.39 KB
/
optimizers.py
File metadata and controls
147 lines (119 loc) · 5.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
# -*- coding: utf-8 -*-
"""
Created on Thu May 25 17:52:56 2023
@author: oowoyele
"""
import torch
import sys
import numpy as np
from scipy.optimize import differential_evolution
from torch.optim.lr_scheduler import StepLR
class OptimizerAdam:
""" Optimizer using Adam for neural networks."""
def __init__(self, fcn=None, parameters=None, learning_rate=0.01):
if parameters is None:
self.parameters = fcn.parameters
else:
self.parameters = parameters
self.optim = torch.optim.Adam(parameters, lr=learning_rate)
def step(self, loss):
"""Take an optimization step."""
self.optim.zero_grad()
loss.backward(retain_graph=True)
self.optim.step()
class OptimizerCLSMAdam:
"""Optimizer using Adam for Competitive Learning Specialized Models."""
def __init__(self, fcn_list=None, parameters=None, learning_rate=0.05):
if fcn_list is not None:
fcn_given = True
self.num_experts = len(fcn_list)
elif parameters is not None:
params_given = True
self.num_experts = len(parameters)
self.optim = []
self.scheduler = []
self.mse_list = [[]] * len(fcn_list)
for iexp in range(self.num_experts):
if fcn_given:
self.parameters = fcn_list[iexp].parameters
elif params_given:
self.parameters = parameters[iexp]
self.optim += [torch.optim.Adam(self.parameters, lr=learning_rate)]
self.scheduler += [StepLR(self.optim[-1], step_size=1000, gamma=0.999)]
if fcn_given:
self.mse_list[iexp] = fcn_list[iexp].mse()
def step(self, loss_list, iter, moe):
"""Take an optimization step for all models."""
for i in range(self.num_experts):
self.optim[i].zero_grad()
loss_list[i].backward(retain_graph=True)
self.optim[i].step()
class OptimizerCLSMNewton:
"""Optimizer using Newton's method for Competitive Learning Specialized Models."""
def __init__(self, fcn_list=None):
self.fcn_list = fcn_list
if fcn_list is not None:
self.num_experts = len(fcn_list)
@staticmethod
def compute_grads(model, alpha):
"""Compute the gradient for a model."""
error = torch.matmul(model.x, model.weights) - model.y
n = model.x.size(0)
gradient_mse = (2 / n) * torch.matmul(model.x.t(), alpha * error)
gradient_l1 = model.lambda_reg * torch.sign(model.weights)
return gradient_mse + gradient_l1
@staticmethod
def compute_hessian(model, alpha):
"""Compute the Hessian matrix for a model."""
n = model.x.size(0)
return (2 / n) * torch.matmul((model.x.t() * alpha.t()), model.x)
def step(self, alpha, learning_rate):
"""Take an optimization step using Newton's method."""
for i in range(len(self.fcn_list)):
model = self.fcn_list[i]
alpha_ = alpha[:,i:i+1]
H = self.compute_hessian(model,alpha_)
g = self.compute_grads(model,alpha_)
diagonal_values = torch.rand(H.shape[0]) * 0.1 #- 1
diagonal_matrix = torch.diag(diagonal_values)
update = torch.matmul(torch.linalg.inv(H + diagonal_matrix), g)
model.weights = model.weights - torch.minimum(torch.tensor(0.001), learning_rate*update)
model.outpt = model.predict()
class GlobalOptimizerCLSM:
"""Global optimizer for Competitive Learning Specialized Models."""
def __init__(self, fcn_list=None):
self.fcn_list = fcn_list
if fcn_list is not None:
self.num_experts = len(fcn_list)
def err(self, weights):
"""Compute the total error for the given weights."""
total_error = 0
start_index = 0
for model in self.fcn_list:
num_weights = np.prod(model.weights.shape)
model_weights = weights[start_index: start_index + num_weights]
model_weights = model_weights.reshape(model.weights.shape)
model.weights = torch.from_numpy(model_weights).type(torch.float64)
total_error += model.mse(update_y=True)
start_index += num_weights
return total_error.detach().numpy()
def global_optimization(self):
"""Perform global optimization on model weights."""
x0 = np.concatenate([model.weights.detach().numpy().ravel() for model in self.fcn_list])
bounds = [(-1, 1) for _ in range(len(x0))]
result = differential_evolution(self.err, bounds, popsize=50, maxiter=3000, disp=False)
optimized_weights = result.x
start_index = 0
for model in self.fcn_list:
num_weights = np.prod(model.weights.shape)
model_weights = optimized_weights[start_index: start_index + num_weights]
model.weights = torch.from_numpy(model_weights).type(torch.float64).reshape(model.weights.shape)
start_index += num_weights
def step(self, alpha, learning_rate):
"""Optimization step for the global optimizer."""
for model in self.fcn_list:
H = OptimizerCLSMNewton.compute_hessian(model, alpha)
g = OptimizerCLSMNewton.compute_grads(model, alpha)
update = torch.matmul(torch.linalg.inv(H), g)
model.weights -= learning_rate * update
model.outpt = model.pred()