src/algos/curiosity.py [175:300]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - plogger = file_writer.FileWriter( xpid=flags.xpid, xp_args=flags.__dict__, rootdir=flags.savedir, ) checkpointpath = os.path.expandvars(os.path.expanduser( '%s/%s/%s' % (flags.savedir, flags.xpid,'model.tar'))) T = flags.unroll_length B = flags.batch_size flags.device = None if not flags.disable_cuda and torch.cuda.is_available(): log.info('Using CUDA.') flags.device = torch.device('cuda') else: log.info('Not using CUDA.') flags.device = torch.device('cpu') env = create_env(flags) if flags.num_input_frames > 1: env = FrameStack(env, flags.num_input_frames) if 'MiniGrid' in flags.env: if flags.use_fullobs_policy: model = FullObsMinigridPolicyNet(env.observation_space.shape, env.action_space.n) else: model = MinigridPolicyNet(env.observation_space.shape, env.action_space.n) if flags.use_fullobs_intrinsic: state_embedding_model = FullObsMinigridStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) else: state_embedding_model = MinigridStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) forward_dynamics_model = MinigridForwardDynamicsNet(env.action_space.n)\ .to(device=flags.device) inverse_dynamics_model = MinigridInverseDynamicsNet(env.action_space.n)\ .to(device=flags.device) else: model = MarioDoomPolicyNet(env.observation_space.shape, env.action_space.n) state_embedding_model = MarioDoomStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) forward_dynamics_model = MarioDoomForwardDynamicsNet(env.action_space.n)\ .to(device=flags.device) inverse_dynamics_model = MarioDoomInverseDynamicsNet(env.action_space.n)\ .to(device=flags.device) buffers = create_buffers(env.observation_space.shape, model.num_actions, flags) model.share_memory() initial_agent_state_buffers = [] for _ in range(flags.num_buffers): state = model.initial_state(batch_size=1) for t in state: t.share_memory_() initial_agent_state_buffers.append(state) actor_processes = [] ctx = mp.get_context('fork') free_queue = ctx.SimpleQueue() full_queue = ctx.SimpleQueue() episode_state_count_dict = dict() train_state_count_dict = dict() for i in range(flags.num_actors): actor = ctx.Process( target=act, args=(i, free_queue, full_queue, model, buffers, episode_state_count_dict, train_state_count_dict, initial_agent_state_buffers, flags)) actor.start() actor_processes.append(actor) if 'MiniGrid' in flags.env: if flags.use_fullobs_policy: learner_model = FullObsMinigridPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) else: learner_model = MinigridPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) else: learner_model = MarioDoomPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) optimizer = torch.optim.RMSprop( learner_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) state_embedding_optimizer = torch.optim.RMSprop( state_embedding_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) inverse_dynamics_optimizer = torch.optim.RMSprop( inverse_dynamics_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) forward_dynamics_optimizer = torch.optim.RMSprop( forward_dynamics_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) def lr_lambda(epoch): return 1 - min(epoch * T * B, flags.total_frames) / flags.total_frames scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) logger = logging.getLogger('logfile') stat_keys = [ 'total_loss', 'mean_episode_return', 'pg_loss', 'baseline_loss', 'entropy_loss', - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - src/algos/ride.py [170:296]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - plogger = file_writer.FileWriter( xpid=flags.xpid, xp_args=flags.__dict__, rootdir=flags.savedir, ) checkpointpath = os.path.expandvars(os.path.expanduser( '%s/%s/%s' % (flags.savedir, flags.xpid,'model.tar'))) T = flags.unroll_length B = flags.batch_size flags.device = None if not flags.disable_cuda and torch.cuda.is_available(): log.info('Using CUDA.') flags.device = torch.device('cuda') else: log.info('Not using CUDA.') flags.device = torch.device('cpu') env = create_env(flags) if flags.num_input_frames > 1: env = FrameStack(env, flags.num_input_frames) if 'MiniGrid' in flags.env: if flags.use_fullobs_policy: model = FullObsMinigridPolicyNet(env.observation_space.shape, env.action_space.n) else: model = MinigridPolicyNet(env.observation_space.shape, env.action_space.n) if flags.use_fullobs_intrinsic: state_embedding_model = FullObsMinigridStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) else: state_embedding_model = MinigridStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) forward_dynamics_model = MinigridForwardDynamicsNet(env.action_space.n)\ .to(device=flags.device) inverse_dynamics_model = MinigridInverseDynamicsNet(env.action_space.n)\ .to(device=flags.device) else: model = MarioDoomPolicyNet(env.observation_space.shape, env.action_space.n) state_embedding_model = MarioDoomStateEmbeddingNet(env.observation_space.shape)\ .to(device=flags.device) forward_dynamics_model = MarioDoomForwardDynamicsNet(env.action_space.n)\ .to(device=flags.device) inverse_dynamics_model = MarioDoomInverseDynamicsNet(env.action_space.n)\ .to(device=flags.device) buffers = create_buffers(env.observation_space.shape, model.num_actions, flags) model.share_memory() initial_agent_state_buffers = [] for _ in range(flags.num_buffers): state = model.initial_state(batch_size=1) for t in state: t.share_memory_() initial_agent_state_buffers.append(state) actor_processes = [] ctx = mp.get_context('fork') free_queue = ctx.SimpleQueue() full_queue = ctx.SimpleQueue() episode_state_count_dict = dict() train_state_count_dict = dict() for i in range(flags.num_actors): actor = ctx.Process( target=act, args=(i, free_queue, full_queue, model, buffers, episode_state_count_dict, train_state_count_dict, initial_agent_state_buffers, flags)) actor.start() actor_processes.append(actor) if 'MiniGrid' in flags.env: if flags.use_fullobs_policy: learner_model = FullObsMinigridPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) else: learner_model = MinigridPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) else: learner_model = MarioDoomPolicyNet(env.observation_space.shape, env.action_space.n)\ .to(device=flags.device) optimizer = torch.optim.RMSprop( learner_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) state_embedding_optimizer = torch.optim.RMSprop( state_embedding_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) inverse_dynamics_optimizer = torch.optim.RMSprop( inverse_dynamics_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) forward_dynamics_optimizer = torch.optim.RMSprop( forward_dynamics_model.parameters(), lr=flags.learning_rate, momentum=flags.momentum, eps=flags.epsilon, alpha=flags.alpha) def lr_lambda(epoch): return 1 - min(epoch * T * B, flags.total_frames) / flags.total_frames scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda) logger = logging.getLogger('logfile') stat_keys = [ 'total_loss', 'mean_episode_return', 'pg_loss', 'baseline_loss', 'entropy_loss', - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -