maga_transformer/_ft_pickler.py [5:110]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    APPEND,
    APPENDS,
    BINFLOAT,
    BINGET,
    BININT,
    BININT1,
    BININT2,
    BINPERSID,
    BINPUT,
    BINUNICODE,
    BUILD,
    bytes_types,
    decode_long,
    EMPTY_DICT,
    EMPTY_LIST,
    EMPTY_SET,
    EMPTY_TUPLE,
    GLOBAL,
    LONG1,
    LONG_BINGET,
    LONG_BINPUT,
    MARK,
    NEWFALSE,
    NEWOBJ,
    NEWTRUE,
    NONE,
    PROTO,
    REDUCE,
    SETITEM,
    SETITEMS,
    SHORT_BINSTRING,
    STOP,
    TUPLE,
    TUPLE1,
    TUPLE2,
    TUPLE3,
    UnpicklingError
)
from struct import unpack
from sys import maxsize
from typing import Any, List, Dict

import torch

class Placeholder():
        def __init__(self, *args: Any, **kwargs: Any):
            pass

        def append(self, _):
            pass
        
        def appends(self, _):
            pass
        
        def update(self, _):
            pass
        
class Unpickler:
    def __init__(self, file: str, *, encoding: str = "bytes"):
        self.encoding = encoding
        self.readline = file.readline
        self.read = file.read
        self.memo: Dict[int, Any] = {}
        self.rc = _get_allowed_globals()
        self.rc['placeholder'] = Placeholder
        
    def load(self):
        """Read a pickled object representation from the open file.

        Return the reconstituted object hierarchy specified in the file.
        """
        self.metastack = []
        self.stack: List[Any] = []
        self.append = self.stack.append
        read = self.read
        readline = self.readline
        while True:
            key = read(1)
            if not key:
                raise EOFError
            assert isinstance(key, bytes_types)
            # Risky operators
            if key[0] == GLOBAL[0]:
                module = readline()[:-1].decode("utf-8")
                name = readline()[:-1].decode("utf-8")
                full_path = f"{module}.{name}"
                if full_path in self.rc:
                    self.append(self.rc[full_path])
                else:
                    self.append(self.rc['placeholder'])
                    # raise RuntimeError(f"Unsupported class {full_path}")
            elif key[0] == NEWOBJ[0]:
                args = self.stack.pop()
                cls = self.stack.pop()
                if cls is not torch.nn.Parameter:
                    self.append(Placeholder(*args))
                    # raise RuntimeError(f"Trying to instantiate unsupported class {cls}")
                else:
                    self.append(torch.nn.Parameter(*args))
            elif key[0] == REDUCE[0]:
                args = self.stack.pop()
                func = self.stack[-1]
                if func not in self.rc.values():
                    raise RuntimeError(
                        f"Trying to call reduce for unrecognized function {func}"
                    )
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



maga_transformer/utils/meta_pickler.py [5:110]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    APPEND,
    APPENDS,
    BINFLOAT,
    BINGET,
    BININT,
    BININT1,
    BININT2,
    BINPERSID,
    BINPUT,
    BINUNICODE,
    BUILD,
    bytes_types,
    decode_long,
    EMPTY_DICT,
    EMPTY_LIST,
    EMPTY_SET,
    EMPTY_TUPLE,
    GLOBAL,
    LONG1,
    LONG_BINGET,
    LONG_BINPUT,
    MARK,
    NEWFALSE,
    NEWOBJ,
    NEWTRUE,
    NONE,
    PROTO,
    REDUCE,
    SETITEM,
    SETITEMS,
    SHORT_BINSTRING,
    STOP,
    TUPLE,
    TUPLE1,
    TUPLE2,
    TUPLE3,
    UnpicklingError
)
from struct import unpack
from sys import maxsize
from typing import Any, List, Dict

import torch

class Placeholder():
        def __init__(self, *args: Any, **kwargs: Any):
            pass

        def append(self, _):
            pass
        
        def appends(self, _):
            pass
        
        def update(self, _):
            pass
        
class Unpickler:
    def __init__(self, file: str, *, encoding: str = "bytes"):
        self.encoding = encoding
        self.readline = file.readline
        self.read = file.read
        self.memo: Dict[int, Any] = {}
        self.rc = _get_allowed_globals()
        self.rc['placeholder'] = Placeholder
        
    def load(self):
        """Read a pickled object representation from the open file.

        Return the reconstituted object hierarchy specified in the file.
        """
        self.metastack = []
        self.stack: List[Any] = []
        self.append = self.stack.append
        read = self.read
        readline = self.readline
        while True:
            key = read(1)
            if not key:
                raise EOFError
            assert isinstance(key, bytes_types)
            # Risky operators
            if key[0] == GLOBAL[0]:
                module = readline()[:-1].decode("utf-8")
                name = readline()[:-1].decode("utf-8")
                full_path = f"{module}.{name}"
                if full_path in self.rc:
                    self.append(self.rc[full_path])
                else:
                    self.append(self.rc['placeholder'])
                    # raise RuntimeError(f"Unsupported class {full_path}")
            elif key[0] == NEWOBJ[0]:
                args = self.stack.pop()
                cls = self.stack.pop()
                if cls is not torch.nn.Parameter:
                    self.append(Placeholder(*args))
                    # raise RuntimeError(f"Trying to instantiate unsupported class {cls}")
                else:
                    self.append(torch.nn.Parameter(*args))
            elif key[0] == REDUCE[0]:
                args = self.stack.pop()
                func = self.stack[-1]
                if func not in self.rc.values():
                    raise RuntimeError(
                        f"Trying to call reduce for unrecognized function {func}"
                    )
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -