maga_transformer/_ft_pickler.py [112:214]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - elif key[0] == BUILD[0]: state = self.stack.pop() inst = self.stack[-1] if type(inst) is torch.Tensor: # Legacy unpickling inst.set_(*state) elif type(inst) is torch.nn.Parameter: inst.__setstate__(state) elif type(inst) is OrderedDict: inst.__dict__.update(state) elif type(inst) is Placeholder: inst.update(state) else: raise RuntimeError( f"Can only build Tensor, parameter or dict objects, but got {type(inst)}" ) # Stack manipulation elif key[0] == APPEND[0]: item = self.stack.pop() list_obj = self.stack[-1] if type(list_obj) is not list: raise RuntimeError( f"Can only append to lists, but got {type(list_obj)}" ) list_obj.append(item) elif key[0] == APPENDS[0]: items = self.pop_mark() list_obj = self.stack[-1] if type(list_obj) is not list: raise RuntimeError( f"Can only extend lists, but got {type(list_obj)}" ) list_obj.extend(items) elif key[0] == SETITEM[0]: (v, k) = (self.stack.pop(), self.stack.pop()) self.stack[-1][k] = v elif key[0] == SETITEMS[0]: items = self.pop_mark() for i in range(0, len(items), 2): self.stack[-1][items[i]] = items[i + 1] elif key[0] == MARK[0]: self.metastack.append(self.stack) self.stack = [] self.append = self.stack.append elif key[0] == TUPLE[0]: items = self.pop_mark() self.append(tuple(items)) elif key[0] == TUPLE1[0]: self.stack[-1] = (self.stack[-1],) elif key[0] == TUPLE2[0]: self.stack[-2:] = [(self.stack[-2], self.stack[-1])] elif key[0] == TUPLE3[0]: self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])] # Basic types construction elif key[0] == NONE[0]: self.append(None) elif key[0] == NEWFALSE[0]: self.append(False) elif key[0] == NEWTRUE[0]: self.append(True) elif key[0] == EMPTY_TUPLE[0]: self.append(()) elif key[0] == EMPTY_LIST[0]: self.append([]) elif key[0] == EMPTY_DICT[0]: self.append({}) elif key[0] == EMPTY_SET[0]: self.append(set()) elif key[0] == BININT[0]: self.append(unpack("d", self.read(8))[0]) elif key[0] == BINUNICODE[0]: strlen = unpack(" maxsize: raise RuntimeError("String is too long") strval = str(read(strlen), "utf-8", "surrogatepass") self.append(strval) elif key[0] == SHORT_BINSTRING[0]: strlen = read(1)[0] strdata = read(strlen) if self.encoding != "bytes": strdata = strdata.decode(self.encoding, "strict") self.append(strdata) elif key[0] == BINPERSID[0]: pid = self.stack.pop() # Only allow persistent load of storage if type(pid) is not tuple and not type(pid) is not int: raise RuntimeError( f"persistent_load id must be tuple or int, but got {type(pid)}" ) if ( type(pid) is tuple and len(pid) > 0 and _maybe_decode_ascii(pid[0]) != "storage" ): raise RuntimeError( f"Only persistent_load of storage is allowed, but got {pid[0]}" ) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - maga_transformer/utils/meta_pickler.py [121:223]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - elif key[0] == BUILD[0]: state = self.stack.pop() inst = self.stack[-1] if type(inst) is torch.Tensor: # Legacy unpickling inst.set_(*state) elif type(inst) is torch.nn.Parameter: inst.__setstate__(state) elif type(inst) is OrderedDict: inst.__dict__.update(state) elif type(inst) is Placeholder: inst.update(state) else: raise RuntimeError( f"Can only build Tensor, parameter or dict objects, but got {type(inst)}" ) # Stack manipulation elif key[0] == APPEND[0]: item = self.stack.pop() list_obj = self.stack[-1] if type(list_obj) is not list: raise RuntimeError( f"Can only append to lists, but got {type(list_obj)}" ) list_obj.append(item) elif key[0] == APPENDS[0]: items = self.pop_mark() list_obj = self.stack[-1] if type(list_obj) is not list: raise RuntimeError( f"Can only extend lists, but got {type(list_obj)}" ) list_obj.extend(items) elif key[0] == SETITEM[0]: (v, k) = (self.stack.pop(), self.stack.pop()) self.stack[-1][k] = v elif key[0] == SETITEMS[0]: items = self.pop_mark() for i in range(0, len(items), 2): self.stack[-1][items[i]] = items[i + 1] elif key[0] == MARK[0]: self.metastack.append(self.stack) self.stack = [] self.append = self.stack.append elif key[0] == TUPLE[0]: items = self.pop_mark() self.append(tuple(items)) elif key[0] == TUPLE1[0]: self.stack[-1] = (self.stack[-1],) elif key[0] == TUPLE2[0]: self.stack[-2:] = [(self.stack[-2], self.stack[-1])] elif key[0] == TUPLE3[0]: self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])] # Basic types construction elif key[0] == NONE[0]: self.append(None) elif key[0] == NEWFALSE[0]: self.append(False) elif key[0] == NEWTRUE[0]: self.append(True) elif key[0] == EMPTY_TUPLE[0]: self.append(()) elif key[0] == EMPTY_LIST[0]: self.append([]) elif key[0] == EMPTY_DICT[0]: self.append({}) elif key[0] == EMPTY_SET[0]: self.append(set()) elif key[0] == BININT[0]: self.append(unpack("d", self.read(8))[0]) elif key[0] == BINUNICODE[0]: strlen = unpack(" maxsize: raise RuntimeError("String is too long") strval = str(read(strlen), "utf-8", "surrogatepass") self.append(strval) elif key[0] == SHORT_BINSTRING[0]: strlen = read(1)[0] strdata = read(strlen) if self.encoding != "bytes": strdata = strdata.decode(self.encoding, "strict") self.append(strdata) elif key[0] == BINPERSID[0]: pid = self.stack.pop() # Only allow persistent load of storage if type(pid) is not tuple and not type(pid) is not int: raise RuntimeError( f"persistent_load id must be tuple or int, but got {type(pid)}" ) if ( type(pid) is tuple and len(pid) > 0 and _maybe_decode_ascii(pid[0]) != "storage" ): raise RuntimeError( f"Only persistent_load of storage is allowed, but got {pid[0]}" ) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -