horovod/keras/__init__.py [34:159]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def DistributedOptimizer(optimizer, name=None, device_dense='', device_sparse='', compression=Compression.none, sparse_as_dense=False, gradient_predivide_factor=1.0): """ An optimizer that wraps another keras.optimizers.Optimizer, using an allreduce to average gradient values before applying gradients to model weights. Args: optimizer: Optimizer to use for computing gradients and applying updates. name: Optional name prefix for the operations created when applying gradients. Defaults to "Distributed" followed by the provided optimizer type. device_dense: Device to be used for dense tensors. Uses GPU by default if Horovod was build with HOROVOD_GPU_OPERATIONS. device_sparse: Device to be used for sparse tensors. Uses GPU by default if Horovod was build with HOROVOD_GPU_OPERATIONS. compression: Compression algorithm used to reduce the amount of data sent and received by each worker node. Defaults to not using compression. sparse_as_dense: Treat all sparse gradients as dense tensors. This can help improve performance and memory utilization if the original sparse gradient has high density. Defaults to false. gradient_predivide_factor: gradient_predivide_factor splits the averaging before and after the sum. Gradients are scaled by 1.0 / gradient_predivide_factor before the sum and gradient_predivide_factor / size after the sum. """ if gradient_predivide_factor != 1.0 and rocm_built(): raise ValueError('gradient_predivide_factor not supported yet with ROCm') return _impl.create_distributed_optimizer(keras, optimizer, name, device_dense, device_sparse, compression, sparse_as_dense, gradient_predivide_factor) def broadcast_global_variables(root_rank): """Broadcasts all global variables from root rank to all other processes. Arguments: root_rank: Rank of the process from which global variables will be broadcasted to all other processes. """ return _impl.broadcast_global_variables(K, root_rank) def allreduce(value, name=None, average=True, prescale_factor=1.0, postscale_factor=1.0): """ Perform an allreduce on a tensor-compatible value. Arguments: value: A tensor-compatible value to reduce. The shape of the input must be identical across all ranks. name: Optional name for the constants created by this operation. average: If True, computes the average over all ranks. Otherwise, computes the sum over all ranks. prescale_factor: Multiplicative factor to scale tensor before allreduce. postscale_factor: Multiplicative factor to scale tensor after allreduce. """ return _impl.allreduce(K, value, name, average, prescale_factor, postscale_factor) def allgather(value, name=None): """ Perform an allgather on a tensor-compatible value. The concatenation is done on the first dimension, so the input values on the different processes must have the same rank and shape, except for the first dimension, which is allowed to be different. Arguments: value: A tensor-compatible value to gather. name: Optional name prefix for the constants created by this operation. """ return _impl.allgather(K, value, name) def broadcast(value, root_rank, name=None): """ Perform a broadcast on a tensor-compatible value. Arguments: value: A tensor-compatible value to reduce. The shape of the input must be identical across all ranks. root_rank: Rank of the process from which global variables will be broadcasted to all other processes. name: Optional name for the constants created by this operation. """ return _impl.broadcast(K, value, root_rank, name) def load_model(filepath, custom_optimizers=None, custom_objects=None, compression=Compression.none): """ Loads a saved Keras model with a Horovod DistributedOptimizer. The DistributedOptimizer will wrap the underlying optimizer used to train the saved model, so that the optimizer state (params and weights) will be picked up for retraining. By default, all optimizers in the module `keras.optimizers` will be loaded and wrapped without needing to specify any `custom_optimizers` or `custom_objects`. Arguments: filepath: One of the following: - string, path to the saved model, or - h5py.File object from which to load the model custom_optimizers: Optional list of Optimizer subclasses to support during loading. custom_objects: Optional dictionary mapping names (strings) to custom classes or functions to be considered during deserialization. compression: Compression algorithm used to reduce the amount of data sent and received by each worker node. Defaults to not using compression. Returns: A Keras model instance. Raises: ImportError: If h5py is not available. ValueError: In case of an invalid savefile. """ def wrap_optimizer(cls): return lambda **kwargs: DistributedOptimizer(cls(**kwargs), compression=compression) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - horovod/tensorflow/keras/__init__.py [47:172]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - def DistributedOptimizer(optimizer, name=None, device_dense='', device_sparse='', compression=Compression.none, sparse_as_dense=False, gradient_predivide_factor=1.0): """ An optimizer that wraps another keras.optimizers.Optimizer, using an allreduce to average gradient values before applying gradients to model weights. Args: optimizer: Optimizer to use for computing gradients and applying updates. name: Optional name prefix for the operations created when applying gradients. Defaults to "Distributed" followed by the provided optimizer type. device_dense: Device to be used for dense tensors. Uses GPU by default if Horovod was build with HOROVOD_GPU_OPERATIONS. device_sparse: Device to be used for sparse tensors. Uses GPU by default if Horovod was build with HOROVOD_GPU_OPERATIONS. compression: Compression algorithm used to reduce the amount of data sent and received by each worker node. Defaults to not using compression. sparse_as_dense: Treat all sparse gradients as dense tensors. This can help improve performance and memory utilization if the original sparse gradient has high density. Defaults to false. gradient_predivide_factor: gradient_predivide_factor splits the averaging before and after the sum. Gradients are scaled by 1.0 / gradient_predivide_factor before the sum and gradient_predivide_factor / size after the sum. """ if gradient_predivide_factor != 1.0 and rocm_built(): raise ValueError('gradient_predivide_factor not supported yet with ROCm') return _impl.create_distributed_optimizer(keras, optimizer, name, device_dense, device_sparse, compression, sparse_as_dense, gradient_predivide_factor) def broadcast_global_variables(root_rank): """Broadcasts all global variables from root rank to all other processes. Arguments: root_rank: Rank of the process from which global variables will be broadcasted to all other processes. """ return _impl.broadcast_global_variables(K, root_rank) def allreduce(value, name=None, average=True, prescale_factor=1.0, postscale_factor=1.0): """ Perform an allreduce on a tensor-compatible value. Arguments: value: A tensor-compatible value to reduce. The shape of the input must be identical across all ranks. name: Optional name for the constants created by this operation. average: If True, computes the average over all ranks. Otherwise, computes the sum over all ranks. prescale_factor: Multiplicative factor to scale tensor before allreduce. postscale_factor: Multiplicative factor to scale tensor after allreduce. """ return _impl.allreduce(K, value, name, average, prescale_factor, postscale_factor) def allgather(value, name=None): """ Perform an allgather on a tensor-compatible value. The concatenation is done on the first dimension, so the input values on the different processes must have the same rank and shape, except for the first dimension, which is allowed to be different. Arguments: value: A tensor-compatible value to gather. name: Optional name prefix for the constants created by this operation. """ return _impl.allgather(K, value, name) def broadcast(value, root_rank, name=None): """ Perform a broadcast on a tensor-compatible value. Arguments: value: A tensor-compatible value to reduce. The shape of the input must be identical across all ranks. root_rank: Rank of the process from which global variables will be broadcasted to all other processes. name: Optional name for the constants created by this operation. """ return _impl.broadcast(K, value, root_rank, name) def load_model(filepath, custom_optimizers=None, custom_objects=None, compression=Compression.none): """ Loads a saved Keras model with a Horovod DistributedOptimizer. The DistributedOptimizer will wrap the underlying optimizer used to train the saved model, so that the optimizer state (params and weights) will be picked up for retraining. By default, all optimizers in the module `keras.optimizers` will be loaded and wrapped without needing to specify any `custom_optimizers` or `custom_objects`. Arguments: filepath: One of the following: - string, path to the saved model, or - h5py.File object from which to load the model custom_optimizers: Optional list of Optimizer subclasses to support during loading. custom_objects: Optional dictionary mapping names (strings) to custom classes or functions to be considered during deserialization. compression: Compression algorithm used to reduce the amount of data sent and received by each worker node. Defaults to not using compression. Returns: A Keras model instance. Raises: ImportError: If h5py is not available. ValueError: In case of an invalid savefile. """ def wrap_optimizer(cls): return lambda **kwargs: DistributedOptimizer(cls(**kwargs), compression=compression) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -