sgd Momentum Vanilla SGD RMSprop adam等优化算法在寻找简单logistic分类中的的应用

本文主要是介绍sgd Momentum Vanilla SGD RMSprop adam等优化算法在寻找简单logistic分类中的的应用，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

参考博文

(4条消息) sgd Momentum Vanilla SGD RMSprop adam等优化算法在寻找函数最值的应用_tcuuuqladvvmm454的博客-CSDN博客

在这里随机选择一些数据生成两类

核心代码如下：
def __init__(self, loss, weights, lr=2.1, beta1=0.9, beta2=0.999, epislon=1e-8):# , t1=[], g1=[], lr1=[], m1=[], v1=[], theta1=[]):
self.loss = loss
self.theta = weights
self.lr = lr
self.beta1 = beta1
self.beta2 = beta2
self.epislon = epislon
self.get_gradient = grad(loss)
self.m = 0
self.v = 0
self.t = 0
#t1=[], g1=[], lr1=[], m1=[], v1=[], theta1=[]
#self.t1, g1, lr1, self.m1, self.v1, self.theta1=[],[],[],[],[],[]

def minimize_raw(self, epochs=EPOCHS):##sgd
ee=[]
#print('sgd------------')
for _ in range(epochs):
self.t += 1
#print('adma------------')
g = self.get_gradient(self.theta)
self.m = self.beta1 * self.m + (1 - self.beta1) * g
self.v = self.beta2 * self.v + (1 - self.beta2) * (g * g)
self.m_cat = self.m / (1 - self.beta1 ** self.t)
self.v_cat = self.v / (1 - self.beta2 ** self.t)
self.theta -= self.lr * self.m_cat / (self.v_cat ** 0.5 + self.epislon)
final_loss = self.loss(self.theta)
ee.append(final_loss)
plt.figure()
plt.plot(ee)
plt.show()
print("adam_final loss:{} weights0:{}".format(final_loss, self.theta[0]))
def minimize_raw1(self, epochs=EPOCHS):##sgd
ee=[]
#print('sgd------------')
for _ in range(epochs):
self.t += 1
g = self.get_gradient(self.theta)
self.m = self.beta1 * self.m + (1 - self.beta1) * g
self.v = self.beta2 * self.v + (1 - self.beta2) * (g * g)
self.m_cat = self.m / (1 - self.beta1 ** self.t)
self.v_cat = self.v / (1 - self.beta2 ** self.t)
self.theta -= self.lr * g#self.m_cat / (self.v_cat ** 0.5 + self.epislon)
final_loss = self.loss(self.theta)
ee.append(final_loss)
plt.figure()
plt.plot(ee)
plt.show()
print("sgd_final loss:{} weights0:{}".format(final_loss, self.theta[0]))

def minimize(self, epochs=EPOCHS):
ee=[]
for _ in range(epochs):

self.t += 1
g = self.get_gradient(self.theta)
lr = self.lr * (1 - self.beta2 ** self.t) ** 0.5 / (1 - self.beta1 ** self.t)
self.m = self.beta1 * self.m + (1 - self.beta1) * g
self.v = self.beta2 * self.v + (1 - self.beta2) * (g * g)
self.theta -= lr * self.m / (self.v ** 0.5 + self.epislon)
final_loss = self.loss(self.theta)
ee.append(final_loss)
plt.figure()
plt.plot(ee)
plt.show()
print("romsrop_final loss:{} weights0:{}".format(final_loss, self.theta[0]))
def minimize2(self, epochs=EPOCHS):
ee=[]
for _ in range(epochs):
self.t += 1
g = self.get_gradient(self.theta)
lr = self.lr * (1 - self.beta2 ** self.t) ** 0.5 / (1 - self.beta1 ** self.t)
self.m = self.beta1 * self.m + lr * g
#self.v = self.beta2 * self.v + (1 - self.beta2) * (g * g)
self.theta -= self.m# / (self.v ** 0.5 + self.epislon)
final_loss = self.loss(self.theta)
ee.append(final_loss)
plt.figure()
plt.plot(ee)
plt.show()
print("dongliang_final loss:{} weights0:{}".format(final_loss, self.theta[0]))

#t1, g1, lr1, m1, v1, theta1=[],[],[],[],[],[]
def minimize_show(self, epochs=EPOCHS):
lr1=[0.1,0.3,0.0001]
for uu in range(3):
for _ in range(epochs):
self.t += 1
lr=lr1[uu]
g = self.get_gradient(self.theta)
lr = self.lr * (1 - self.beta2 ** self.t) ** 0.5 / (1 - self.beta1 ** self.t)
self.m = self.beta1 * self.m + (1 - self.beta1) * g
self.v = self.beta2 * self.v + (1 - self.beta2) * (g * g)
self.theta -= lr * self.m / (self.v ** 0.5 + self.epislon)
#print("step{: 4d} g:{} lr:{} m:{} v:{} theta:{}".format(self.t, g, lr, self.m, self.v, self.theta))
#t1.append(self.t)
#g1.append(g),
#l.r1append(l.r)
#m1.append(self.m)
#v1.append(self.v)
#theta1.append(self.theta)#=[],[],[],[],[],[]
#return self.t1, g1, lr1, self.m1, self.v1, self.theta1

final_loss = self.loss(self.theta)
print("final loss:{} weights:{}".format(final_loss, self.theta))

def sigmoid(x):

return 1/(np.exp(-x) + 1)#0.5*(np.tanh(x) + 1)

def plot_sigmoid_dao( ):
x=np.arange(-8,8,0.1)
y=sigmoid(x)*(1-sigmoid(x))
y1=sigmoid(x)#*(1-sigmoid(x))
p1=plt.plot(x,y,label='sigmod1 ')
p2=plt.plot(x,y1,label='sigmod')
plt.legend( )#[p2, p1], ["yuanshi2", "daosgu 1"], loc='upper left')
plt.show()

#plt.legend('daoshu','yuanshi')
plot_sigmoid_dao()

def logistic_predictions(weights, inputs):
# Outputs probability of a label being true according to logistic model.
return sigmoid(np.dot(inputs, weights))

def training_loss1(weights):
rr=[]
# Training loss is the negative log-likelihood of the training labels.
preds = logistic_predictions(weights, inputs)
rr.append((preds))
#print(rr)
label_probabilities = (preds - targets)**2#preds * targets + (1 - preds) * (1 - targets)

#return -np.sum(np.log(label_probabilities))
return np.sum((label_probabilities))/preds.shape[0]

def training_loss(weights):
rr=[]
s1=np.dot(inputs, weights)
s2=sigmoid(s1)
#print('s2=',s2)
# Training loss is the negative log-likelihood of the training labels.
preds = s2#logistic_predictions(weights, inputs)
rr.append((preds))
#print(preds ,targets)
ee=[]
for i in range(preds.shape[0]):
ee.append((preds[i]-targets[i])**2)
ee1=sum(ee)

#label_probabilities = (preds - targets)**2#preds * targets + (1 - preds) * (1 - targets)
#print(label_probabilities)
#return -np.sum(np.log(label_probabilities))
return ee1#np.sum((label_probabilities))