in eden/scm/edenscm/hgext/fsmonitor/__init__.py [0:0]
def _innerwalk(self, match, event, span):
state = self._fsmonitorstate
clock, ignorehash, notefiles = state.get()
if not clock:
if state.walk_on_invalidate:
raise fsmonitorfallback("no clock")
# Initial NULL clock value, see
# https://facebook.github.io/watchman/docs/clockspec.html
clock = "c:0:0"
notefiles = []
ignore = self.dirstate._ignore
# experimental config: experimental.fsmonitor.skipignore
if not self._ui.configbool("experimental", "fsmonitor.skipignore"):
if ignorehash and _hashignore(ignore) != ignorehash and clock != "c:0:0":
# ignore list changed -- can't rely on Watchman state any more
if state.walk_on_invalidate:
raise fsmonitorfallback("ignore rules changed")
notefiles = []
clock = "c:0:0"
matchfn = match.matchfn
matchalways = match.always()
dmap = self.dirstate._map
if util.safehasattr(dmap, "_map"):
# for better performance, directly access the inner dirstate map if the
# standard dirstate implementation is in use.
dmap = dmap._map
if "treestate" in self._repo.requirements:
# treestate has a fast path to filter out ignored directories.
ignorevisitdir = self.dirstate._ignore.visitdir
def dirfilter(path):
result = ignorevisitdir(path.rstrip("/"))
return result == "all"
nonnormalset = self.dirstate._map.nonnormalsetfiltered(dirfilter)
else:
nonnormalset = self.dirstate._map.nonnormalset
event["old_clock"] = clock
event["old_files"] = blackbox.shortlist(sorted(nonnormalset))
span.record(oldclock=clock, oldfileslen=len(nonnormalset))
state.setlastnonnormalfilecount(len(nonnormalset))
copymap = self.dirstate._map.copymap
getkind = stat.S_IFMT
dirkind = stat.S_IFDIR
regkind = stat.S_IFREG
lnkkind = stat.S_IFLNK
join = self.dirstate._join
normcase = util.normcase
fresh_instance = False
exact = False
if match.isexact(): # match.exact
exact = True
if not exact and self.dirstate._checkcase:
# note that even though we could receive directory entries, we're only
# interested in checking if a file with the same name exists. So only
# normalize files if possible.
normalize = self.dirstate._normalizefile
else:
normalize = None
# step 2: query Watchman
with progress.spinner(self._ui, "watchman query"):
try:
# Use the user-configured timeout for the query.
# Add a little slack over the top of the user query to allow for
# overheads while transferring the data
excludes = ["anyof", ["dirname", ".hg"], ["name", ".hg", "wholename"]]
# Exclude submodules.
if git.isgitformat(self._repo):
submods = git.parsesubmodules(self._repo[None])
excludes += [["dirname", s.path] for s in submods]
self._watchmanclient.settimeout(state.timeout + 0.1)
result = self._watchmanclient.command(
"query",
{
"fields": ["mode", "mtime", "size", "exists", "name"],
"since": clock,
"expression": ["not", excludes],
"sync_timeout": int(state.timeout * 1000),
"empty_on_fresh_instance": state.walk_on_invalidate,
},
)
except Exception as ex:
event["is_error"] = True
span.record(error=ex)
_handleunavailable(self._ui, state, ex)
self._watchmanclient.clearconnection()
# XXX: Legacy scuba logging. Remove this once the source of truth
# is moved to the Rust Event.
self._ui.log("fsmonitor_status", fsmonitor_status="exception")
if self._ui.configbool("fsmonitor", "fallback-on-watchman-exception"):
raise fsmonitorfallback("exception during run")
else:
raise ex
else:
# We need to propagate the last observed clock up so that we
# can use it for our next query
event["new_clock"] = result["clock"]
event["is_fresh"] = result["is_fresh_instance"]
span.record(newclock=result["clock"], isfresh=result["is_fresh_instance"])
state.setlastclock(result["clock"])
state.setlastisfresh(result["is_fresh_instance"])
files = list(
filter(lambda x: _isutf8(self._ui, x["name"]), result["files"])
)
if result["is_fresh_instance"]:
if not self._ui.plain() and self._ui.configbool(
"fsmonitor", "warn-fresh-instance"
):
oldpid = _watchmanpid(event["old_clock"])
newpid = _watchmanpid(event["new_clock"])
if oldpid is not None and newpid is not None and oldpid != newpid:
self._ui.warn(
_(
"warning: watchman has recently restarted (old pid %s, new pid %s) - operation will be slower than usual\n"
)
% (oldpid, newpid)
)
elif oldpid is None and newpid is not None:
self._ui.warn(
_(
"warning: watchman has recently started (pid %s) - operation will be slower than usual\n"
)
% (newpid,)
)
else:
self._ui.warn(
_(
"warning: watchman failed to catch up with file change events and requires a full scan - operation will be slower than usual\n"
)
)
if state.walk_on_invalidate:
state.invalidate(reason="fresh_instance")
raise fsmonitorfallback("fresh instance")
fresh_instance = True
# Ignore any prior noteable files from the state info
notefiles = []
else:
count = len(files)
state.setwatchmanchangedfilecount(count)
event["new_files"] = blackbox.shortlist(
sorted(e["name"] for e in files), count
)
span.record(newfileslen=len(files))
# XXX: Legacy scuba logging. Remove this once the source of truth
# is moved to the Rust Event.
if event["is_fresh"]:
self._ui.log("fsmonitor_status", fsmonitor_status="fresh")
else:
self._ui.log("fsmonitor_status", fsmonitor_status="normal")
results = {}
# for file paths which require normalization and we encounter a case
# collision, we store our own foldmap
if normalize:
foldmap = dict((normcase(k), k) for k in results)
switch_slashes = pycompat.ossep == "\\"
# The order of the results is, strictly speaking, undefined.
# For case changes on a case insensitive filesystem we may receive
# two entries, one with exists=True and another with exists=False.
# The exists=True entries in the same response should be interpreted
# as being happens-after the exists=False entries due to the way that
# Watchman tracks files. We use this property to reconcile deletes
# for name case changes.
ignorelist = []
ignorelistappend = ignorelist.append
with progress.bar(self.ui, _("Watchman results"), _("files"), len(files)) as prog:
for entry in files:
prog.value += 1
fname = entry["name"]
if _fixencoding:
fname = _watchmantofsencoding(fname)
if switch_slashes:
fname = fname.replace("\\", "/")
if normalize:
normed = normcase(fname)
fname = normalize(fname, True, True)
foldmap[normed] = fname
fmode = entry["mode"]
fexists = entry["exists"]
kind = getkind(fmode)
if not fexists:
# if marked as deleted and we don't already have a change
# record, mark it as deleted. If we already have an entry
# for fname then it was either part of walkexplicit or was
# an earlier result that was a case change
if (
fname not in results
and fname in dmap
and (matchalways or matchfn(fname))
):
results[fname] = None
elif kind == dirkind:
if fname in dmap and (matchalways or matchfn(fname)):
results[fname] = None
elif kind == regkind or kind == lnkkind:
if fname in dmap:
if matchalways or matchfn(fname):
results[fname] = entry
else:
ignored = ignore(fname)
if ignored:
ignorelistappend(fname)
if (matchalways or matchfn(fname)) and not ignored:
results[fname] = entry
elif fname in dmap and (matchalways or matchfn(fname)):
results[fname] = None
elif fname in match.files():
match.bad(fname, filesystem.badtype(kind))
# step 3: query notable files we don't already know about
# XXX try not to iterate over the entire dmap
if normalize:
# any notable files that have changed case will already be handled
# above, so just check membership in the foldmap
notefiles = set(
(normalize(f, True, True) for f in notefiles if normcase(f) not in foldmap)
)
visit = set(
(
f
for f in notefiles
if (f not in results and matchfn(f) and (f in dmap or not ignore(f)))
)
)
if not fresh_instance:
if matchalways:
visit.update(f for f in nonnormalset if f not in results)
visit.update(f for f in copymap if f not in results)
else:
visit.update(f for f in nonnormalset if f not in results and matchfn(f))
visit.update(f for f in copymap if f not in results and matchfn(f))
else:
if matchalways:
visit.update(f for f in dmap if f not in results)
visit.update(f for f in copymap if f not in results)
else:
visit.update(f for f in dmap if f not in results and matchfn(f))
visit.update(f for f in copymap if f not in results and matchfn(f))
# audit returns False for paths with one of its parent directories being a
# symlink.
audit = pathutil.pathauditor(self.dirstate._root, cached=True).check
with progress.bar(self.ui, _("Auditing paths"), _("files"), len(visit)) as prog:
auditpass = []
for f in visit:
prog.value += 1
if audit(f):
auditpass.append(f)
auditpass.sort()
auditfail = visit.difference(auditpass)
droplist = []
droplistappend = droplist.append
for f in auditfail:
# For auditfail paths, they should be treated as not existed in working
# copy.
filestate = dmap.get(f, ("?", 0, 0, 0))[0]
if filestate in ("?",):
# do not exist in working parents, remove them from treestate and
# avoid walking through them.
droplistappend(f)
results.pop(f, None)
else:
# tracked, mark as deleted
results[f] = None
auditpassiter = iter(auditpass)
def nf():
return next(auditpassiter)
with progress.bar(
self.ui, _("Getting metadata"), _("files"), len(auditpass)
) as prog:
# Break it into chunks so we get some progress information
for i in range(0, len(auditpass), 5000):
chunk = auditpass[i : i + 5000]
for st in util.statfiles([join(f) for f in chunk]):
prog.value += 1
f = nf()
if (st and not ignore(f)) or f in dmap:
results[f] = st
elif not st:
# '?' (untracked) file was deleted from the filesystem - remove it
# from treestate.
#
# We can only update the dirstate (and treestate) while holding the
# wlock. That happens inside poststatus.__call__ -> state.set. So
# buffer what files to "drop" so state.set can clean them up.
entry = dmap.get(f, None)
if entry and entry[0] == "?":
droplistappend(f)
# The droplist and ignorelist need to match setlastclock()
state.setdroplist(droplist)
state.setignorelist(ignorelist)
results.pop(".hg", None)
return pycompat.iteritems(results)