func()

in maintner/gerrit.go [1029:1184]

• 130 lines of code
• 24 McCabe index (conditional complexity)

func (gp *GerritProject) syncOnce(ctx context.Context) error {
	if err := gp.syncMissingCommits(ctx); err != nil {
		return err
	}

	c := gp.gerrit.c
	gitDir := gp.gitDir()

	t0 := time.Now()
	cmd := exec.CommandContext(ctx, "git", "fetch", "origin")
	envutil.SetDir(cmd, gitDir)
	// Enable extra Git tracing in case the fetch hangs.
	envutil.SetEnv(cmd,
		"GIT_TRACE2_EVENT=1",
		"GIT_TRACE_CURL_NO_DATA=1",
	)
	cmd.Stdout = new(bytes.Buffer)
	cmd.Stderr = cmd.Stdout

	// The 'git fetch' needs a timeout in case it hangs, but to avoid spurious
	// timeouts (and live-lock) the timeout should be (at least) an order of
	// magnitude longer than we expect the operation to actually take. Moreover,
	// exec.CommandContext sends SIGKILL, which may terminate the command without
	// giving it a chance to flush useful trace entries, so we'll terminate it
	// manually instead (see https://golang.org/issue/22757).
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("git fetch origin: %v", err)
	}
	timer := time.AfterFunc(10*time.Minute, func() {
		cmd.Process.Signal(os.Interrupt)
	})
	err := cmd.Wait()
	fetchDuration := time.Since(t0).Round(time.Millisecond)
	timer.Stop()
	if err != nil {
		return fmt.Errorf("git fetch origin: %v after %v, %s", err, fetchDuration, cmd.Stdout)
	}
	gp.logf("ran git fetch origin in %v", fetchDuration)

	t0 = time.Now()
	cmd = exec.CommandContext(ctx, "git", "ls-remote")
	envutil.SetDir(cmd, gitDir)
	out, err := cmd.CombinedOutput()
	lsRemoteDuration := time.Since(t0).Round(time.Millisecond)
	if err != nil {
		return fmt.Errorf("git ls-remote in %s: %v after %v, %s", gitDir, err, lsRemoteDuration, out)
	}
	gp.logf("ran git ls-remote in %v", lsRemoteDuration)

	var changedRefs []*maintpb.GitRef
	var toFetch []GitHash

	bs := bufio.NewScanner(bytes.NewReader(out))

	// Take the lock here to access gp.remote and call c.gitHashFromHex.
	// It's acceptable to take such a coarse-looking lock because
	// it's not actually around I/O: all the input from ls-remote has
	// already been slurped into memory.
	c.mu.Lock()
	refExists := map[string]bool{} // whether ref is this ls-remote fetch
	for bs.Scan() {
		line := bs.Bytes()
		tab := bytes.IndexByte(line, '\t')
		if tab == -1 {
			if !strings.HasPrefix(bs.Text(), "From ") {
				gp.logf("bogus ls-remote line: %q", line)
			}
			continue
		}
		sha1 := string(line[:tab])
		refName := strings.TrimSpace(string(line[tab+1:]))
		refExists[refName] = true
		hash := c.gitHashFromHexStr(sha1)

		var needFetch bool

		m := rxRemoteRef.FindSubmatch(line)
		if m != nil {
			clNum, err := strconv.ParseInt(string(m[2]), 10, 32)
			version, ok := gerritVersionNumber(string(m[3]))
			if err != nil || !ok {
				continue
			}
			curHash := gp.remote[gerritCLVersion{int32(clNum), version}]
			needFetch = curHash != hash
		} else if trackGerritRef(refName) && gp.ref[refName] != hash {
			needFetch = true
			gp.logf("ref %q = %q", refName, sha1)
		}

		if needFetch {
			toFetch = append(toFetch, hash)
			changedRefs = append(changedRefs, &maintpb.GitRef{
				Ref:  refName,
				Sha1: string(sha1),
			})
		}
	}
	var deletedRefs []string
	for n := range gp.ref {
		if !refExists[n] {
			gp.logf("ref %q now deleted", n)
			deletedRefs = append(deletedRefs, n)
		}
	}
	c.mu.Unlock()

	if err := bs.Err(); err != nil {
		gp.logf("ls-remote scanning error: %v", err)
		return err
	}
	if len(deletedRefs) > 0 {
		c.addMutation(&maintpb.Mutation{
			Gerrit: &maintpb.GerritMutation{
				Project:     gp.proj,
				DeletedRefs: deletedRefs,
			},
		})
	}
	if len(changedRefs) == 0 {
		return nil
	}
	gp.logf("%d new refs", len(changedRefs))
	const batchSize = 250
	for len(toFetch) > 0 {
		batch := toFetch
		if len(batch) > batchSize {
			batch = batch[:batchSize]
		}
		if err := gp.fetchHashes(ctx, batch); err != nil {
			return err
		}

		c.mu.Lock()
		for _, hash := range batch {
			gp.markNeededCommit(hash)
		}
		c.mu.Unlock()

		n, err := gp.syncCommits(ctx)
		if err != nil {
			return err
		}
		toFetch = toFetch[len(batch):]
		gp.logf("synced %v commits for %d new hashes, %d hashes remain", n, len(batch), len(toFetch))

		c.addMutation(&maintpb.Mutation{
			Gerrit: &maintpb.GerritMutation{
				Project: gp.proj,
				Refs:    changedRefs[:len(batch)],
			}})
		changedRefs = changedRefs[len(batch):]
	}

	return nil
}