in aliyun/log/etl_core/trans_comp/trans_json.py [0:0]
def __init__(self, jmes=None, jmes_ignore_none=None, output=None,
expand=None, depth=None, include_node=None, exclude_node=None,
include_path=None, exclude_path=None,
fmt=None, sep=None, prefix=None, suffix=None,
expand_array=None, fmt_array=None, mode=None
):
"""
:param jmes: jmes filter to select or generate new field
:param jmes_ignore_none: if jmes filter is null, ignore it (default). Or else consider it as "". default is
:param output: put the value parsed from jmes filter to this field
:param expand: If jmes filter is configure, expand the result or not (Default is False in this case), If jmes is not configured, directly expand the field passed or not (Default is True in this case).
:param depth: depth to scan, 1 means first layer, default is 100.
:param include_node: keys to expand and include. regex string. using '|' for multiple ones. default is all.
:param exclude_node: keys to skip, regex string. using '|' for multiple ones. default is nothing.
:param include_path: path to expand and include. regex string to match from begin. using '|' for multiple ones. default is all. e.g. r"data\.k1", all sub-keys in data.k1 will be included.
:param exclude_path: path to skip, regex string to match from begin. using '|' for multiple ones. default is nothing. . e.g. r"data\.k2", all sub-keys in data.k2 will be excluded.
:param fmt: during expansion, how to format the key name, there're several types or customized as described in FMT_MAP
:param sep: sep used in formatting during expansion
:param prefix: prefix used in formatting during expansion
:param suffix: suffix used in formatting during expansion
:param expand_array: if expand array or just render it. default is True. item in array will be with name index, e.g. [1,2,3] will be considered as {"0": 1, "1": 2, "2": 3}
:param fmt_array: format string for key name of each array element, default is "{parent_rlist[0]}_{index}", can be custom formatting string using placehodler: parent_list, parent_list, current
"""
super(json_transformer, self).__init__(mode=mode)
self.expand = expand
if expand is None:
# when jmes is not configured or configure but no output configured
self.expand = not jmes or not output
# self.level = level or 1
self.jmes = self._u(jmes or "")
self.prefix = self._u("" if prefix is None else prefix)
self.suffix = self._u("" if suffix is None else suffix)
self.sep = self._u(self.DEFAULT_SEP if sep is None else sep)
self.output = self._u(output or "")
self.jmes_filter = None
self.jmes_ignore_none = True if jmes_ignore_none is None else jmes_ignore_none
if jmes:
try:
self.jmes_filter = jmespath.compile(jmes)
except jmespath.exceptions.ParseError as ex:
raise SettingError(ex=ex, msg="Invalid JMES filter setting", settings=jmes)
elif self.output:
logger.warning(u"json_transformer: parameter output '{0}' will be ignored as there's no filter is selected."
.format(output))
self.depth = min((depth or self.DEFAULT_DEPTH), self.DEFAULT_DEPTH)
self.include_node = self._u(include_node or self.DEFAULT_INCLUDE_NODE)
self.exclude_node = self._u(exclude_node or self.DEFAULT_EXCLUDE_NODE)
self.include_path = self._u(include_path or self.DEFAULT_INCLUDE_PATH)
self.exclude_path = self._u(exclude_path or self.DEFAULT_EXCLUDE_PATH)
self.fmt = self._u(fmt or self.DEFAULT_FMT)
try:
self.include_node_match = get_re_full_match(self.include_node)
self.exclude_node_match = get_re_full_match(self.exclude_node)
self.include_path_match = re.compile(self.include_path).match # use match instead of full match
self.exclude_path_match = re.compile(self.exclude_path).match # use match instead of full match
except Exception as ex:
raise SettingError(ex=ex, msg="Invalid regex string for include/exclude")
self.expand_array = True if expand_array is None else expand_array
self.format_array = self._u(fmt_array or self.DEFAULT_FMT_ARRAY)