in src/google/appengine/datastore/datastore_stub_index.py [0:0]
def UpdateIndexYaml(self, openfile=open):
"""Update index.yaml.
Args:
openfile: Used for dependency injection.
We only ever write to index.yaml if either:
- it doesn't exist yet; or
- it contains an 'AUTOGENERATED' comment.
All indexes *before* the AUTOGENERATED comment will be written
back unchanged. All indexes *after* the AUTOGENERATED comment
will be updated with the latest query counts (query counts are
reset by --clear_datastore). Indexes that aren't yet in the file
will be appended to the AUTOGENERATED section.
We keep track of some data in order to avoid doing repetitive work:
- if index.yaml is fully manual, we keep track of its mtime to
avoid parsing it over and over;
- we keep track of the number of keys in the history dict since
the last time we updated index.yaml (or decided there was
nothing to update).
"""
index_yaml_file = os.path.join(self.root_path, 'index.yaml')
try:
index_yaml_mtime = os.path.getmtime(index_yaml_file)
except os.error:
index_yaml_mtime = None
index_yaml_changed = (index_yaml_mtime != self.index_yaml_mtime)
self.index_yaml_mtime = index_yaml_mtime
datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
query_ci_history_len = datastore_stub._QueryCompositeIndexHistoryLength()
history_changed = (query_ci_history_len != self.last_history_size)
self.last_history_size = query_ci_history_len
if not (index_yaml_changed or history_changed):
logger.debug('No need to update index.yaml')
return
if self.index_yaml_is_manual and not index_yaml_changed:
logger.debug('Will not update manual index.yaml')
return
if index_yaml_mtime is None:
index_yaml_data = None
else:
try:
# The 'U' option is deprecated in py3, as universal newlines
# are the default option
fh = openfile(index_yaml_file, 'r' if six.PY3 else 'rU')
except IOError:
index_yaml_data = None
else:
try:
index_yaml_data = fh.read()
if not index_yaml_data:
raise EmptyIndexFileError(
'The index yaml file is empty. The file should at least have '
'an empty "indexes:" block.')
finally:
fh.close()
self.index_yaml_is_manual = (index_yaml_data is not None and
AUTO_MARKER not in index_yaml_data)
if self.index_yaml_is_manual:
logger.info('Detected manual index.yaml, will not update')
return
if index_yaml_data is None:
all_indexes = None
else:
try:
all_indexes = datastore_index.ParseIndexDefinitions(index_yaml_data)
except yaml_errors.EventListenerError as e:
logger.error('Error parsing %s:\n%s', index_yaml_file, e)
return
except Exception as err:
logger.error('Error parsing %s:\n%s.%s: %s', index_yaml_file,
err.__class__.__module__, err.__class__.__name__, err)
return
if index_yaml_data is None:
manual_part, prev_automatic_part = 'indexes:\n', ''
manual_indexes = None
else:
manual_part, prev_automatic_part = index_yaml_data.split(AUTO_MARKER, 1)
if prev_automatic_part.startswith(AUTO_COMMENT):
prev_automatic_part = prev_automatic_part[len(AUTO_COMMENT):]
try:
manual_indexes = datastore_index.ParseIndexDefinitions(manual_part)
except Exception as err:
logger.error('Error parsing manual part of %s: %s',
index_yaml_file, err)
return
automatic_part = GenerateIndexFromHistory(datastore_stub.QueryHistory(),
all_indexes, manual_indexes)
if (index_yaml_mtime is None and automatic_part == '' or
automatic_part == prev_automatic_part):
logger.debug('No need to update index.yaml')
return
try:
fh = openfile(index_yaml_file, 'w')
except IOError as err:
logger.error('Can\'t write index.yaml: %s', err)
return
try:
logger.info('Updating %s', index_yaml_file)
fh.write(manual_part)
fh.write(AUTO_MARKER)
fh.write(AUTO_COMMENT)
fh.write(automatic_part)
finally:
fh.close()
try:
self.index_yaml_mtime = os.path.getmtime(index_yaml_file)
except os.error as err:
logger.error('Can\'t stat index.yaml we just wrote: %s', err)
self.index_yaml_mtime = None