btf/btfhive/getFuncs.py [182:453]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - cur.execute(sql) sql = """CREATE TABLE members ( id INTEGER PRIMARY KEY autoincrement, fid INTEGER, types VARCHAR (128), name VARCHAR (64), offset INTEGER, bytes INTEGER, bits VARCHAR (16) DEFAULT "" );""" cur.execute(sql) sql = """CREATE TABLE types ( id INTEGER PRIMARY KEY autoincrement, name VARCHAR (64), alias VARCHAR (64), bytes INTEGER );""" cur.execute(sql) cur.close() def _arg_split(self, argStr): args = [] arg = "" count = 0 for a in argStr: if count == 0 and a == ",": args.append(arg.strip()) arg = "" continue elif a == "(": count += 1 elif a == ")": count -= 1 arg += a if arg != "": args.append(arg.strip()) return args def _funcs(self, funcPath, module="vm"): cur = self._db.cursor() with open(funcPath, 'r') as f: fid = 0 for index, line in enumerate(f): line = line[:-1] if line == "": continue elif line.startswith("(gdb)"): break elif line.startswith("File "): if line.endswith(".h:"): # do not add any fid = -1 else: _, sFile = line.split(" ", 1) sql = f'''INSERT INTO files (file) VALUES ("{sFile[:-1]}")''' cur.execute(sql) fid = cur.lastrowid elif fid > 0 and line.endswith(");"): #8: static int __paravirt_pgd_alloc(struct mm_struct *); line = line[:-2] lineNo, body = line.split(":", 1) head, args = body.split("(", 1) # args = [x.strip() for x in args.split(",")] args = self._arg_split(args) if "*" in head: ret, func = head.rsplit("*", 1) ret += "*" else: ret, func = head.rsplit(" ", 1) sql = f'''INSERT INTO funs (func, args, ret, line, fid, module) VALUES \ ("{func}", '{json.dumps(args)}', "{ret.strip()}", {lineNo}, {fid}, "{module}")''' cur.execute(sql) cur.close() def _stripRem(self, line): return self._reRem.sub("", line).strip() def _splitStructLine(self, line): rd = {"offset": None, "size": None, "bits": None} res = self._rePaholeRem1.search(line) if res: l = res.group()[2:-2].strip() # /* 19: 0 | 1 */ unsigned char skc_reuse : 4; # /* 19: 4 | 1 */ unsigned char skc_reuseport : 1; off, size = l.split('|', 1) rd["size"] = int(size.strip()) if ":" in off: off, bits = off.split(":", 1) rd['bits'] = bits.strip() # offset rd["offset"] = int(off.strip()) else: res = self._rePaholeRem2.search(line) if res: l = res.group()[2:-2].strip() # /* 8 | 4 */ union { # /* 4 */ unsigned int skc_hash; # /* 4 */ __u16 skc_u16hashes[2]; size = l.strip() rd["size"] = int(size) rd["line"] = self._stripRem(line) return rd def _parseMember(self, sStruct, line, pre="", off=0): """struct list_head * next;""" """void (*func)(struct callback_head *);""" """unsigned int p:1;""" if ";" not in line: """/* total size (bytes): 4 */""" if "total size (bytes):" in line: size = line.split(":", 1)[1] size = size.split("*", 1)[0].strip() self._res['res']['size'] = size return rd = self._splitStructLine(line) l = rd['line'] bits = "" if ':' in l: l, bits = l.rsplit(" : ", 1) bits = "%s:%s" % (rd["bits"], bits) if '(' in l: _, func = l.split("(*", 1) func, _ = func.split(")", 1) types = l.replace(" (*%s)(" % func, " (*)(", 1) types = re.sub(" +", " ", types) name = func elif '*' in l: types, name = l.rsplit("*", 1) types = types + "*" name = name.strip("; ") else: types, name = l.rsplit(" ", 1) types = types.strip() name = name.strip("; ") name = pre + name if rd["offset"] is None: rd["offset"] = off cell = {"type": types, "name": name, "offset": rd["offset"], "size": rd["size"], "bits": bits} self._res['res']['cell'].append(cell) def _parseBox(self, sStruct, lines, pre): """union {""" """} pci;""" rd = self._splitStructLine(lines[0]) t = rd['line'].split(" ", 1)[0] if t in ["union", "struct"]: lastLine = lines[-1].strip() if not lastLine.startswith("};"): npre, _ = lastLine[1:].split(";", 1) _, npre = npre.rsplit(" ", 1) pre += npre.strip() + "." if rd["offset"] is None: rd["offset"] = 0 self._parseLoop(sStruct, lines, pre, rd["offset"]) def _parseLoop(self, sStruct, lines, pre, off=0): qCount = 0 box = [] for line in lines[1:-1]: lCount = line.count("{") rCount = line.count("}") qCount += lCount - rCount if qCount > 0: box.append(line) elif len(box) > 0: box.append(line) self._parseBox(sStruct, box, pre) box = [] else: self._parseMember(sStruct, line, pre, off) def _getStruct(self, gdb, sStruct): self._res = {"log": "struct"} lines = gdb.showStruct(sStruct).split('\n') self._res['res'] = {"name": sStruct, "size": 0, "cell": []} self._parseLoop(sStruct, lines, "") self._res['res']['members'] = len(self._res['res']['cell']) return self._res def _struct_is_in(self, cur, sStruct): sql = f"SELECT name FROM structs WHERE name = '{sStruct}'" res = cur.execute(sql) if res is None: return False r = res.fetchone() if r is None: return False return True def _struct(self, cur, gdb, sStruct): if self._struct_is_in(cur, sStruct): return res = self._getStruct(gdb, sStruct) if res is None: return dStruct = res['res'] sql = f'''INSERT INTO structs (name, members, bytes) VALUES \ ("{dStruct['name']}", {dStruct['members']}, {dStruct['size']})''' cur.execute(sql) fid = cur.lastrowid for cell in dStruct['cell']: sql = f'''INSERT INTO members (fid, types, name, offset, bytes, bits) VALUES \ ({fid}, "{cell['type']}", "{cell['name']}", {cell['offset']}, {cell['size']}, "{cell['bits']}")''' try: cur.execute(sql) except sqlite3.OperationalError: print(f"bad {sql}, for {dStruct['name']}") def _save_type(self, cur, gdb, t): alias, size = gdb.genType(t) sql = f'INSERT INTO types (name, alias, bytes) VALUES ("{t}", "{alias}", {size})' cur.execute(sql) if alias == "struct {" or alias == 'union {': # there is no alias struct in this type self._struct(cur, gdb, t) def _type_is_in(self, cur, t): if t in self._banTypes: return True sql = f"SELECT name FROM types WHERE name = '{t}'" res = cur.execute(sql) if res is None: return False r = res.fetchone() if r is None: return False return True def _check_type(self, cur, gdb, t): if not self._type_is_in(cur, t): try: self._save_type(cur, gdb, t) except ValueError: self._banTypes.append(t) print(f"failed to parse type {t}") def _types(self, typePath, cur, gdb): with open(typePath, 'r') as f: for i, line in enumerate(f): if i < 1 or line.startswith("(gdb)"): # skip head and end continue if self._reTypeLine.match(line): line = line.split(':', 1)[1] line = line.strip() if line != "": # strip blank line if line.startswith("File "): # jump File continue if line.startswith("enum "): continue if line.startswith("struct ") or line.startswith("union "): self._struct(cur, gdb, line[:-1]) continue if line.startswith("typedef "): # for typedef sType = line.rsplit(" ", 1)[1] self._check_type(cur, gdb, sType[:-1]) # skip last ; continue if line.startswith("__int128"): line = "__int128" self._check_type(cur, gdb, line) def _parseElf(self, cur, gdb, mod): try: gdb.genTypes("types.txt") gdb.genFuncs("funcs.txt") except OSError as e: print(f"parse error. report {e}") return self._funcs("funcs.txt", mod) self._types("types.txt", cur, gdb) def pasrseVmLinux(self, vmPath): cur = self._db.cursor() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - compile/remote-compile/lbc/tool/getFuncs.py [186:457]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - cur.execute(sql) sql = """CREATE TABLE members ( id INTEGER PRIMARY KEY autoincrement, fid INTEGER, types VARCHAR (128), name VARCHAR (64), offset INTEGER, bytes INTEGER, bits VARCHAR (16) DEFAULT "" );""" cur.execute(sql) sql = """CREATE TABLE types ( id INTEGER PRIMARY KEY autoincrement, name VARCHAR (64), alias VARCHAR (64), bytes INTEGER );""" cur.execute(sql) cur.close() def _arg_split(self, argStr): args = [] arg = "" count = 0 for a in argStr: if count == 0 and a == ",": args.append(arg.strip()) arg = "" continue elif a == "(": count += 1 elif a == ")": count -= 1 arg += a if arg != "": args.append(arg.strip()) return args def _funcs(self, funcPath, module="vm"): cur = self._db.cursor() with open(funcPath, 'r') as f: fid = 0 for index, line in enumerate(f): line = line[:-1] if line == "": continue elif line.startswith("(gdb)"): break elif line.startswith("File "): if line.endswith(".h:"): # do not add any fid = -1 else: _, sFile = line.split(" ", 1) sql = f'''INSERT INTO files (file) VALUES ("{sFile[:-1]}")''' cur.execute(sql) fid = cur.lastrowid elif fid > 0 and line.endswith(");"): #8: static int __paravirt_pgd_alloc(struct mm_struct *); line = line[:-2] lineNo, body = line.split(":", 1) head, args = body.split("(", 1) # args = [x.strip() for x in args.split(",")] args = self._arg_split(args) if "*" in head: ret, func = head.rsplit("*", 1) ret += "*" else: ret, func = head.rsplit(" ", 1) sql = f'''INSERT INTO funs (func, args, ret, line, fid, module) VALUES \ ("{func}", '{json.dumps(args)}', "{ret.strip()}", {lineNo}, {fid}, "{module}")''' cur.execute(sql) cur.close() def _stripRem(self, line): return self._reRem.sub("", line).strip() def _splitStructLine(self, line): rd = {"offset": None, "size": None, "bits": None} res = self._rePaholeRem1.search(line) if res: l = res.group()[2:-2].strip() # /* 19: 0 | 1 */ unsigned char skc_reuse : 4; # /* 19: 4 | 1 */ unsigned char skc_reuseport : 1; off, size = l.split('|', 1) rd["size"] = int(size.strip()) if ":" in off: off, bits = off.split(":", 1) rd['bits'] = bits.strip() # offset rd["offset"] = int(off.strip()) else: res = self._rePaholeRem2.search(line) if res: l = res.group()[2:-2].strip() # /* 8 | 4 */ union { # /* 4 */ unsigned int skc_hash; # /* 4 */ __u16 skc_u16hashes[2]; size = l.strip() rd["size"] = int(size) rd["line"] = self._stripRem(line) return rd def _parseMember(self, sStruct, line, pre="", off=0): """struct list_head * next;""" """void (*func)(struct callback_head *);""" """unsigned int p:1;""" if ";" not in line: """/* total size (bytes): 4 */""" if "total size (bytes):" in line: size = line.split(":", 1)[1] size = size.split("*", 1)[0].strip() self._res['res']['size'] = size return rd = self._splitStructLine(line) l = rd['line'] bits = "" if ':' in l: l, bits = l.rsplit(" : ", 1) bits = "%s:%s" % (rd["bits"], bits) if '(' in l: _, func = l.split("(*", 1) func, _ = func.split(")", 1) types = l.replace(" (*%s)(" % func, " (*)(", 1) types = re.sub(" +", " ", types) name = func elif '*' in l: types, name = l.rsplit("*", 1) types = types + "*" name = name.strip("; ") else: types, name = l.rsplit(" ", 1) types = types.strip() name = name.strip("; ") name = pre + name if rd["offset"] is None: rd["offset"] = off cell = {"type": types, "name": name, "offset": rd["offset"], "size": rd["size"], "bits": bits} self._res['res']['cell'].append(cell) def _parseBox(self, sStruct, lines, pre): """union {""" """} pci;""" rd = self._splitStructLine(lines[0]) t = rd['line'].split(" ", 1)[0] if t in ["union", "struct"]: lastLine = lines[-1].strip() if not lastLine.startswith("};"): npre, _ = lastLine[1:].split(";", 1) _, npre = npre.rsplit(" ", 1) pre += npre.strip() + "." if rd["offset"] is None: rd["offset"] = 0 self._parseLoop(sStruct, lines, pre, rd["offset"]) def _parseLoop(self, sStruct, lines, pre, off=0): qCount = 0 box = [] for line in lines[1:-1]: lCount = line.count("{") rCount = line.count("}") qCount += lCount - rCount if qCount > 0: box.append(line) elif len(box) > 0: box.append(line) self._parseBox(sStruct, box, pre) box = [] else: self._parseMember(sStruct, line, pre, off) def _getStruct(self, gdb, sStruct): self._res = {"log": "struct"} lines = gdb.showStruct(sStruct).split('\n') self._res['res'] = {"name": sStruct, "size": 0, "cell": []} self._parseLoop(sStruct, lines, "") self._res['res']['members'] = len(self._res['res']['cell']) return self._res def _struct_is_in(self, cur, sStruct): sql = f"SELECT name FROM structs WHERE name = '{sStruct}'" res = cur.execute(sql) if res is None: return False r = res.fetchone() if r is None: return False return True def _struct(self, cur, gdb, sStruct): if self._struct_is_in(cur, sStruct): return res = self._getStruct(gdb, sStruct) if res is None: return dStruct = res['res'] sql = f'''INSERT INTO structs (name, members, bytes) VALUES \ ("{dStruct['name']}", {dStruct['members']}, {dStruct['size']})''' cur.execute(sql) fid = cur.lastrowid for cell in dStruct['cell']: sql = f'''INSERT INTO members (fid, types, name, offset, bytes, bits) VALUES \ ({fid}, "{cell['type']}", "{cell['name']}", {cell['offset']}, {cell['size']}, "{cell['bits']}")''' try: cur.execute(sql) except sqlite3.OperationalError: print(f"bad {sql}, for {dStruct['name']}") def _save_type(self, cur, gdb, t): alias, size = gdb.genType(t) sql = f'INSERT INTO types (name, alias, bytes) VALUES ("{t}", "{alias}", {size})' cur.execute(sql) if alias == "struct {" or alias == 'union {': # there is no alias struct in this type self._struct(cur, gdb, t) def _type_is_in(self, cur, t): if t in self._banTypes: return True sql = f"SELECT name FROM types WHERE name = '{t}'" res = cur.execute(sql) if res is None: return False r = res.fetchone() if r is None: return False return True def _check_type(self, cur, gdb, t): if not self._type_is_in(cur, t): try: self._save_type(cur, gdb, t) except ValueError: self._banTypes.append(t) print(f"failed to parse type {t}") def _types(self, typePath, cur, gdb): with open(typePath, 'r') as f: for i, line in enumerate(f): if i < 1 or line.startswith("(gdb)"): # skip head and end continue if self._reTypeLine.match(line): line = line.split(':', 1)[1] line = line.strip() if line != "": # strip blank line if line.startswith("File "): # jump File continue if line.startswith("enum "): continue if line.startswith("struct ") or line.startswith("union "): self._struct(cur, gdb, line[:-1]) continue if line.startswith("typedef "): # for typedef sType = line.rsplit(" ", 1)[1] self._check_type(cur, gdb, sType[:-1]) # skip last ; continue if line.startswith("__int128"): line = "__int128" self._check_type(cur, gdb, line) def _parseElf(self, cur, gdb, mod): try: gdb.genTypes("types.txt") gdb.genFuncs("funcs.txt") except OSError as e: print(f"parse error. report {e}") return self._funcs("funcs.txt", mod) self._types("types.txt", cur, gdb) def pasrseVmLinux(self, vmPath): cur = self._db.cursor() - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -