in ridge_sketch.py [0:0]
def iterations_accel_sketch_proj(self, b, w, sketch_method, m):
if self.mu == 0 or self.nu == 0:
# srt = self.sketch_size / m
# mu, nu = (srt *0.25 , 4.0 / srt)
# mu, nu = (srt ** (4 / 5), 1.0 / (srt ** (5 / 4)))
mu, nu = 0.1, 5.0
elif self.mu == "theorydense" or self.nu == "theorydense":
diags = sketch_method.A.get_diag()
eigs = np.linalg.eigvalsh(sketch_method.A._A)
lmin = np.min(eigs)
trA = np.sum(diags)
minAii = np.min(diags)
mu = lmin / trA # (self.alpha+minAii)/(trA+minAii)
nu = trA / minAii
else:
mu = self.mu
nu = self.nu
# mu, nu = self.set_aggressive_accel_parameters(A)
gamma = np.sqrt(1 / (mu * nu)) # step size
a = 1 / (1 + gamma * nu) # momentum parameter alpha
beta = 1 - np.sqrt(mu / nu) # momentum parameter
v = w.copy() # averaged estimate
z = w.copy() # extrapolated estimate
rw = -b.copy() # initialize residual when w=0, which is -b
rz = -b.copy() # residual of z
rv = -b.copy() # residual of v
self.residual_norms.append(1.0)
r_norm_initial = np.linalg.norm(rw)
rw_norm = r_norm_initial
for i in range(self.max_iter):
if self.verbose and (i % 50 == 0):
print(f"iter:{i:^8d} | rel res norm: {(rw_norm / r_norm_initial):.2e}")
SA, SAS, rs = sketch_method.sketch(rz) # Sketch A, AS and rz
lmbda = self.solve_system(SAS, rs)
z = a * v + (1 - a) * w # 1st momentum step
w = z.copy() # Every other workaround failed :( I can explain
sketch_method.update_iterate(w, lmbda) # update w (SGD step)
v = beta * v + (1 - beta) * z + gamma * (w - z) # 2nd mom step
rw = rz - safe_sparse_dot(SA.T, lmbda) # updating the residuals
rv = (
beta * rv + (1 - beta) * rz + gamma * (rw - rz)
) # updating the residuals
rz = a * rv + (1 - a) * rw # updating the residuals
# COMMENT. Both z and w converge.
# Choosing the one with minimum residual:
rz_norm = np.linalg.norm(rz)
rw_norm = np.linalg.norm(rw)
# err = np.minimum(rw_norm, rz_norm)/r_norm_initial
err = rw_norm / r_norm_initial
self.residual_norms.append(err)
if err < self.tol:
break
self.iterations = i
if rz_norm < rw_norm:
return z
else:
return w