in internal/worker/fetch.go [109:271]
func (f *Fetcher) FetchAndUpdateState(ctx context.Context, modulePath, requestedVersion, appVersionLabel string) (status int, resolvedVersion string, err error) {
defer derrors.Wrap(&err, "FetchAndUpdateState(%q, %q, %q)", modulePath, requestedVersion, appVersionLabel)
tctx, span := trace.StartSpan(ctx, "FetchAndUpdateState")
ctx = experiment.NewContext(tctx, experiment.FromContext(ctx).Active()...)
ctx = log.NewContextWithLabel(ctx, "fetch", modulePath+"@"+requestedVersion)
start := time.Now()
var nPackages int64
defer func() {
latency := float64(time.Since(start).Seconds())
dcensus.RecordWithTag(ctx, dcensus.KeyStatus, strconv.Itoa(status), fetchLatency.M(latency))
if status < 300 {
stats.Record(ctx, fetchedPackages.M(nPackages))
}
}()
if !utf8.ValidString(modulePath) {
log.Errorf(ctx, "module path %q is not valid UTF-8", modulePath)
}
if modulePath == internal.UnknownModulePath {
return http.StatusInternalServerError, "", errors.New("called with internal.UnknownModulePath")
}
if !utf8.ValidString(requestedVersion) {
log.Errorf(ctx, "requested version %q is not valid UTF-8", requestedVersion)
}
span.AddAttributes(
trace.StringAttribute("modulePath", modulePath),
trace.StringAttribute("version", requestedVersion))
defer span.End()
// Begin by htting the proxy's info endpoint. We need the resolved version
// to do load-shedding, but it's also important to make the proxy aware
// of the version if it isn't already, as can happen when we arrive here via
// frontend fetch.
//
// Don't fail on a non-nil error. If we return here, we won't record
// the error state in the DB.
info, err := getInfo(ctx, modulePath, requestedVersion, f.ProxyClient)
if err == nil {
// If we're overloaded, shed load by not processing this module.
// The zip endpoint requires a resolved version.
deferFunc, zipSize, err := f.maybeShed(ctx, modulePath, info.Version)
defer deferFunc()
if err != nil {
return derrors.ToStatus(err), "", err
}
fi := &FetchInfo{
ModulePath: modulePath,
Version: requestedVersion,
ZipSize: uint64(zipSize),
Start: time.Now(),
}
startFetchInfo(fi)
defer func() { finishFetchInfo(fi, status, err) }()
// If this is a valid module, insert it into module_version_states.
//
// In case something happens later on, this will make sure we retry. Also,
// modules that are introduced to pkgsite for the first time via frontend
// fetch and not index.golang.org won't have a row in
// module_version_states, so that ensures the logic below works properly as
// well.
//
// Leave the index_timestamp as empty. This will be populated when the
// module appears in the index.
if err := f.DB.InsertNewModuleVersionFromFrontendFetch(ctx, modulePath, info.Version); err != nil {
return derrors.ToStatus(err), "", err
}
}
// Get the latest-version information first, and update the DB. We'll need
// it to determine if the current module version is the latest good one for
// its path.
lmv, err := f.FetchAndUpdateLatest(ctx, modulePath)
// The only errors possible here should be DB failures.
if err != nil {
return derrors.ToStatus(err), "", err
}
ft := f.fetchAndInsertModule(ctx, modulePath, requestedVersion, lmv)
nPackages = int64(len(ft.PackageVersionStates))
span.AddAttributes(trace.Int64Attribute("numPackages", nPackages))
// If there were any errors processing the module then we didn't insert it.
// Delete it in case we are reprocessing an existing module.
// However, don't delete if the error was internal, or we are shedding load.
if ft.Status >= 400 && ft.Status < 500 {
if err := deleteModule(ctx, f.DB, ft); err != nil {
log.Error(ctx, err)
ft.Error = err
ft.Status = http.StatusInternalServerError
}
}
// Regardless of what the status code is, insert the result into
// version_map, so that a response can be returned for frontend_fetch.
if err := updateVersionMap(ctx, f.DB, ft); err != nil {
log.Error(ctx, err)
if ft.Status != http.StatusInternalServerError {
ft.Error = err
ft.Status = http.StatusInternalServerError
}
// Do not return an error here, because we want to insert into
// module_version_states below.
}
if !semver.IsValid(ft.ResolvedVersion) {
// If the requestedVersion was not successfully resolved to a semantic
// version, then at this point it will be the same as the
// resolvedVersion. This fetch request does not need to be recorded in
// module_version_states, since that table is only used to track
// modules that have been published to index.golang.org.
return ft.Status, ft.ResolvedVersion, ft.Error
}
// Return an error here if a row does not exist in module_version_states.
// This can happen if the source is frontend fetch, since we don't insert
// rows to avoid cluttering module_version_states.
if _, err := f.DB.GetModuleVersionState(ctx, modulePath, ft.ResolvedVersion); err != nil {
if errors.Is(err, derrors.NotFound) {
return ft.Status, "", ft.Error
}
return http.StatusInternalServerError, "", err
}
// Make sure the latest version of the module is the one in search_documents
// and imports_unique.
if err := f.DB.ReconcileSearch(ctx, modulePath, ft.ResolvedVersion, ft.Status); err != nil {
log.Error(ctx, err)
if ft.Status != http.StatusInternalServerError {
ft.Error = err
ft.Status = http.StatusInternalServerError
}
// Do not return an error here, because we want to insert into
// module_version_states below.
}
// Update the module_version_states table with the new status of
// module@version. This must happen last, because if it succeeds with a
// code < 500 but a later action fails, we will never retry the later
// action.
startUpdate := time.Now()
mvs := &postgres.ModuleVersionStateForUpdate{
ModulePath: ft.ModulePath,
Version: ft.ResolvedVersion,
AppVersion: appVersionLabel,
Status: ft.Status,
HasGoMod: ft.HasGoMod,
GoModPath: ft.GoModPath,
FetchErr: ft.Error,
PackageVersionStates: ft.PackageVersionStates,
}
err = f.DB.UpdateModuleVersionState(ctx, mvs)
ft.timings["db.UpdateModuleVersionState"] = time.Since(startUpdate)
if err != nil {
log.Error(ctx, err)
if ft.Error != nil {
ft.Status = http.StatusInternalServerError
ft.Error = fmt.Errorf("db.UpdateModuleVersionState: %v, original error: %v", err, ft.Error)
}
logTaskResult(ctx, ft, "Failed to update module version state")
return http.StatusInternalServerError, ft.ResolvedVersion, ft.Error
}
logTaskResult(ctx, ft, "Updated module version state")
return ft.Status, ft.ResolvedVersion, ft.Error
}