MaskRCNN/pytorch/maskrcnn_benchmark/layers/deconv.py [428:510]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        return x




def isqrt_newton_schulz_autograd(A, numIters,norm='norm',method='inverse_newton'):
    dim = A.shape[0]

    if norm=='norm':
        normA=A.norm()
    else:
        normA=A.trace()

    I = torch.eye(dim, dtype=A.dtype, device=A.device)
    Y = A.div(normA)

    Z = torch.eye(dim, dtype=A.dtype, device=A.device)

    if method=='denman_beavers':
        for i in range(numIters):
            #T = 0.5*(3.0*I - Z@Y)
            T=torch.addmm(beta=1.5, input=I, alpha=-0.5, mat1=Z, mat2=Y)
            Y = Y.mm(T)
            Z = T.mm(Z)
    elif method=='newton':
        for i in range(numIters):
            #Z =  1.5 * Z - 0.5* Z@ Z @ Z @ Y
            Z = torch.addmm(beta=1.5, input=Z, alpha=-0.5, mat1=torch.matrix_power(Z, 3), mat2=Y)
    elif method=='inverse_newton':
        for i in range(numIters):
            T= (3*I - Y)/2
            Y = torch.mm(torch.matrix_power(T, 2),Y)
            Z = Z.mm(T)

    #A_sqrt = Y* torch.sqrt(normA)
    A_isqrt =Z/ torch.sqrt(normA)
    return A_isqrt


def isqrt_newton_schulz_autograd_batch(A, numIters,method='inverse_newton'):
    batchSize,dim,_ = A.shape
    normA=A.view(batchSize, -1).norm(2, 1).view(batchSize, 1, 1)
    Y = A.div(normA)
    I = torch.eye(dim,dtype=A.dtype,device=A.device).unsqueeze(0).expand_as(A)
    Z = torch.eye(dim,dtype=A.dtype,device=A.device).unsqueeze(0).expand_as(A)
    if method=='denman_beavers':
        for i in range(numIters):
            T = 0.5*(3.0*I - Z.bmm(Y))
            Y = Y.bmm(T)
            Z = T.bmm(Z)
    elif method=='inverse_newton':
        for i in range(numIters):
            T= (3*I - Y)/2
            Y = torch.bmm(torch.matrix_power(T, 2),Y)
            Z = Z.bmm(T)
    #A_sqrt = Y*torch.sqrt(normA)
    A_isqrt = Z / torch.sqrt(normA)

    return A_isqrt



class LayerNorm(nn.Module):
    def __init__(self, eps=1e-4):
        super(LayerNorm, self).__init__()
        self.eps=eps

    def forward(self,x):
        x_shape=x.shape
        x=x.reshape(x_shape[0],-1)
        mean = x.mean(-1,keepdim=True)
        std = x.std(-1,keepdim=True)+ self.eps
        #x = (x - mean) / std #disaster
        x = x /std- mean/std#this is way more efficient
        x=x.view(x_shape)
        return x




if __name__=='__main__':

    pass
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



Segmentation/models/segmentation/deconv.py [275:356]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        return x



def isqrt_newton_schulz_autograd(A, numIters,norm='norm',method='inverse_newton'):
    dim = A.shape[0]

    if norm=='norm':
        normA=A.norm()
    else:
        normA=A.trace()

    I = torch.eye(dim, dtype=A.dtype, device=A.device)
    Y = A.div(normA)

    Z = torch.eye(dim, dtype=A.dtype, device=A.device)

    if method=='denman_beavers':
        for i in range(numIters):
            #T = 0.5*(3.0*I - Z@Y)
            T=torch.addmm(beta=1.5, input=I, alpha=-0.5, mat1=Z, mat2=Y)
            Y = Y.mm(T)
            Z = T.mm(Z)
    elif method=='newton':
        for i in range(numIters):
            #Z =  1.5 * Z - 0.5* Z@ Z @ Z @ Y
            Z = torch.addmm(beta=1.5, input=Z, alpha=-0.5, mat1=torch.matrix_power(Z, 3), mat2=Y)
    elif method=='inverse_newton':
        for i in range(numIters):
            T= (3*I - Y)/2
            Y = torch.mm(torch.matrix_power(T, 2),Y)
            Z = Z.mm(T)

    #A_sqrt = Y* torch.sqrt(normA)
    A_isqrt =Z/ torch.sqrt(normA)
    return A_isqrt


def isqrt_newton_schulz_autograd_batch(A, numIters,method='inverse_newton'):
    batchSize,dim,_ = A.shape
    normA=A.view(batchSize, -1).norm(2, 1).view(batchSize, 1, 1)
    Y = A.div(normA)
    I = torch.eye(dim,dtype=A.dtype,device=A.device).unsqueeze(0).expand_as(A)
    Z = torch.eye(dim,dtype=A.dtype,device=A.device).unsqueeze(0).expand_as(A)
    if method=='denman_beavers':
        for i in range(numIters):
            T = 0.5*(3.0*I - Z.bmm(Y))
            Y = Y.bmm(T)
            Z = T.bmm(Z)
    elif method=='inverse_newton':
        for i in range(numIters):
            T= (3*I - Y)/2
            Y = torch.bmm(torch.matrix_power(T, 2),Y)
            Z = Z.bmm(T)
    #A_sqrt = Y*torch.sqrt(normA)
    A_isqrt = Z / torch.sqrt(normA)

    return A_isqrt



class LayerNorm(nn.Module):
    def __init__(self, eps=1e-4):
        super(LayerNorm, self).__init__()
        self.eps=eps

    def forward(self,x):
        x_shape=x.shape
        x=x.reshape(x_shape[0],-1)
        mean = x.mean(-1,keepdim=True)
        std = x.std(-1,keepdim=True)+ self.eps
        #x = (x - mean) / std #disaster
        x = x /std- mean/std#this is way more efficient
        x=x.view(x_shape)
        return x




if __name__=='__main__':

    pass
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -