fairscale/experimental/nn/data_parallel/gossip/distributed.py [690:708]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        offset = 0
        for group in optimizer.param_groups:
            for p in group["params"]:
                numel = p.numel()

                if offset + numel > self.portion_start and offset < self.portion_end:

                    # start and end for each
                    overall_start = max(self.portion_start, offset)
                    overall_end = min(self.portion_end, offset + numel)

                    p_start = overall_start - offset
                    p_end = overall_end - offset

                    buffer_start = overall_start - self.portion_start
                    buffer_end = overall_end - self.portion_start

                    # let's see size of p and split based on that
                    current_p = p.view(-1)[p_start:p_end]
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



fairscale/experimental/nn/data_parallel/gossip/distributed.py [775:794]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            offset = 0
            for group in optimizer.param_groups:
                # perform local slowmo for p
                for p in group["params"]:
                    numel = p.numel()

                    if offset + numel > self.portion_start and offset < self.portion_end:

                        # start and end for each
                        overall_start = max(self.portion_start, offset)
                        overall_end = min(self.portion_end, offset + numel)

                        p_start = overall_start - offset
                        p_end = overall_end - offset

                        buffer_start = overall_start - self.portion_start
                        buffer_end = overall_end - self.portion_start

                        # let's see size of p and split based on that
                        current_p = p.view(-1)[p_start:p_end]
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -