MTRF/algorithms/softlearning/algorithms/multi_sac.py [691:735]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        feed_dict = self._get_feed_dict(iteration, batch)

        self._session.run(self._training_ops_per_policy[self._goal_index], feed_dict)

        if self._rnd_int_rew_coeffs[self._goal_index]:
            int_rew_std = np.maximum(np.std(self._session.run(
                self._unscaled_int_rewards[self._goal_index], feed_dict)), 1e-3)
            self._running_int_rew_stds[
                self._goal_index] = self._running_int_rew_stds[self._goal_index] * self._rnd_gamma + int_rew_std * (1-self._rnd_gamma)

        if self._normalize_ext_reward_gamma != 1 and self._ext_reward_coeffs[self._goal_index]:
            ext_rew_std = np.maximum(np.std(self._session.run(
                self._normalized_ext_rewards[self._goal_index], feed_dict)), 1e-3)
            self._running_ext_rew_stds[
                self._goal_index] = self._running_ext_rew_stds[self._goal_index] * self._normalize_ext_reward_gamma + \
                ext_rew_std * (1-self._normalize_ext_reward_gamma)

        self._num_grad_steps_taken_per_policy[self._goal_index] += 1

        if self._num_grad_steps_taken_per_policy[self._goal_index] % self._target_update_interval == 0:
            # Run target ops here.
            self._update_target(self._goal_index)

    def _get_feed_dict(self, iteration, batch):
        batch_flat = flatten(batch)
        placeholders_flat = flatten(self._placeholders)

        # if np.random.rand() < 1e-4 and 'pixels' in batch['observations']:
        #     import os
        #     from skimage import io
        #     random_idx = np.random.randint(
        #         batch['observations']['pixels'].shape[0])
        #     image_save_dir = os.path.join(os.getcwd(), 'pixels')
        #     image_save_path = os.path.join(
        #         image_save_dir, f'observation_{iteration}_batch.png')
        #     if not os.path.exists(image_save_dir):
        #         os.makedirs(image_save_dir)
        #     io.imsave(image_save_path,
        #               batch['observations']['pixels'][random_idx].copy())

        feed_dict = {
            placeholders_flat[key]: batch_flat[key]
            for key in placeholders_flat.keys()
            if key in batch_flat.keys()
        }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



MTRF/algorithms/softlearning/algorithms/phased_sac.py [503:547]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        feed_dict = self._get_feed_dict(iteration, batch)

        self._session.run(self._training_ops_per_policy[self._goal_index], feed_dict)

        if self._rnd_int_rew_coeffs[self._goal_index]:
            int_rew_std = np.maximum(np.std(self._session.run(
                self._unscaled_int_rewards[self._goal_index], feed_dict)), 1e-3)
            self._running_int_rew_stds[
                self._goal_index] = self._running_int_rew_stds[self._goal_index] * self._rnd_gamma + int_rew_std * (1-self._rnd_gamma)

        if self._normalize_ext_reward_gamma != 1 and self._ext_reward_coeffs[self._goal_index]:
            ext_rew_std = np.maximum(np.std(self._session.run(
                self._normalized_ext_rewards[self._goal_index], feed_dict)), 1e-3)
            self._running_ext_rew_stds[
                self._goal_index] = self._running_ext_rew_stds[self._goal_index] * self._normalize_ext_reward_gamma + \
                ext_rew_std * (1-self._normalize_ext_reward_gamma)

        self._num_grad_steps_taken_per_policy[self._goal_index] += 1

        if self._num_grad_steps_taken_per_policy[self._goal_index] % self._target_update_interval == 0:
            # Run target ops here.
            self._update_target(self._goal_index)

    def _get_feed_dict(self, iteration, batch):
        batch_flat = flatten(batch)
        placeholders_flat = flatten(self._placeholders)

        # if np.random.rand() < 1e-4 and 'pixels' in batch['observations']:
        #     import os
        #     from skimage import io
        #     random_idx = np.random.randint(
        #         batch['observations']['pixels'].shape[0])
        #     image_save_dir = os.path.join(os.getcwd(), 'pixels')
        #     image_save_path = os.path.join(
        #         image_save_dir, f'observation_{iteration}_batch.png')
        #     if not os.path.exists(image_save_dir):
        #         os.makedirs(image_save_dir)
        #     io.imsave(image_save_path,
        #               batch['observations']['pixels'][random_idx].copy())

        feed_dict = {
            placeholders_flat[key]: batch_flat[key]
            for key in placeholders_flat.keys()
            if key in batch_flat.keys()
        }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -