in tensorflow_gan/python/losses/losses_impl.py [0:0]
def combine_adversarial_loss(main_loss,
adversarial_loss,
weight_factor=None,
gradient_ratio=None,
gradient_ratio_epsilon=1e-6,
variables=None,
scalar_summaries=True,
gradient_summaries=True,
scope=None):
"""Utility to combine main and adversarial losses.
This utility combines the main and adversarial losses in one of two ways.
1) Fixed coefficient on adversarial loss. Use `weight_factor` in this case.
2) Fixed ratio of gradients. Use `gradient_ratio` in this case. This is often
used to make sure both losses affect weights roughly equally, as in
https://arxiv.org/pdf/1705.05823.
One can optionally also visualize the scalar and gradient behavior of the
losses.
Args:
main_loss: A float Tensor of any shape, indicating the main loss. The size
of the first dimension must be the same as the first dimension of
adversarial_loss. If main_loss and adversarial_loss are not compatible
shapes, both will be mean-reduced to just their first dimension (assumed
to be the batch dimension).
adversarial_loss: A float Tensor of any shape, indicating the adversarial
loss. The size of the first dimension must be the same as the first
dimension of main_loss. If main_loss and adversarial_loss are not
compatible shapes, both will be mean-reduced to just their first dimension
(assumed to be the batch dimension).
weight_factor: If not `None`, the coefficient by which to multiply the
adversarial loss. Exactly one of this and `gradient_ratio` must be
non-None.
gradient_ratio: If not `None`, the ratio of the magnitude of the gradients.
Specifically,
gradient_ratio = grad_mag(main_loss) / grad_mag(adversarial_loss)
Exactly one of this and `weight_factor` must be non-None.
gradient_ratio_epsilon: An epsilon to add to the adversarial loss
coefficient denominator, to avoid division-by-zero.
variables: List of variables to calculate gradients with respect to. If not
present, defaults to all trainable variables.
scalar_summaries: Create scalar summaries of losses. If main_loss and
adversarial_loss are not scalars, they will be mean-reduced to scalars for
summary computation.
gradient_summaries: Create gradient summaries of losses.
scope: Optional name scope.
Returns:
A float Tensor indicating the desired combined loss. If main_loss and
adversarial_loss are both scalars then this will also be a scalar, otherwise
it will be of shape [main_loss.shape[0]].
Raises:
ValueError: Malformed input.
RuntimeError: If `tf.gradients` require computing, and TensorFlow is
executing eagerly.
"""
_validate_args(weight_factor, gradient_ratio)
if variables is None:
variables = contrib.get_trainable_variables()
with tf.compat.v1.name_scope(
scope, 'adversarial_loss', values=[main_loss, adversarial_loss]):
# If losses are not the same shape, reduce them to both be shape [batch,].
if not main_loss.shape.is_compatible_with(adversarial_loss.shape):
if main_loss.shape[0] != adversarial_loss.shape[0]:
raise ValueError(
'main_loss and adversarial_loss must have the same sized first '
'dimension. Found %d and %d.' %
(main_loss.shape[0], adversarial_loss.shape[0]))
tf.compat.v1.logging.warning(
'Applying mean reduction per-batch-element to main and adversarial '
'losses to make shapes compatible. If this is undesirable, ensure '
'that the shapes are compatible before passing them into '
'combine_adversarial_loss.')
main_loss = tf.math.reduce_mean(
input_tensor=main_loss, axis=list(range(1, main_loss.shape.rank)))
adversarial_loss = tf.math.reduce_mean(
input_tensor=adversarial_loss,
axis=list(range(1, adversarial_loss.shape.rank)))
# Compute gradients if we will need them.
if gradient_summaries or gradient_ratio is not None:
# `tf.gradients` doesn't work in eager.
if tf.executing_eagerly():
raise RuntimeError('`tf.gradients` doesn\'t work in eager.')
main_loss_grad_mag = numerically_stable_global_norm(
tf.gradients(ys=main_loss, xs=variables))
adv_loss_grad_mag = numerically_stable_global_norm(
tf.gradients(ys=adversarial_loss, xs=variables))
# Add summaries, if applicable.
if scalar_summaries:
tf.compat.v1.summary.scalar('main_loss',
tf.math.reduce_mean(input_tensor=main_loss))
tf.compat.v1.summary.scalar(
'adversarial_loss',
tf.math.reduce_mean(input_tensor=adversarial_loss))
if gradient_summaries:
tf.compat.v1.summary.scalar('main_loss_gradients', main_loss_grad_mag)
tf.compat.v1.summary.scalar('adversarial_loss_gradients',
adv_loss_grad_mag)
# Combine losses in the appropriate way.
# If `weight_factor` is always `0`, avoid computing the adversarial loss
# tensor entirely.
if _used_weight((weight_factor, gradient_ratio)) == 0:
final_loss = main_loss
elif weight_factor is not None:
final_loss = (main_loss +
tf.stop_gradient(weight_factor) * adversarial_loss)
elif gradient_ratio is not None:
grad_mag_ratio = main_loss_grad_mag / (
adv_loss_grad_mag + gradient_ratio_epsilon)
adv_coeff = grad_mag_ratio / gradient_ratio
tf.compat.v1.summary.scalar('adversarial_coefficient', adv_coeff)
final_loss = (main_loss +
tf.stop_gradient(adv_coeff) * adversarial_loss)
return final_loss