in optimum/graphcore/ipu_configuration.py [0:0]
def _validate_ipu_config(self):
"""
Tests coherence of `IPUConfig` attributes for all modes
in self.modes. For example if `matmul_proportion=[0.2, 0.2]`,
`ipus_per_replica` must have value 2.
Raises:
IncompatibleIPUConfigError: Raised if any `IPUConfig` attributes are not coherent.
"""
if self.replicated_tensor_sharding and self.replication_factor == 1:
logger.warning("`replicated_tensor_sharding` is not used when `replication_factor=1`")
old_mode = self.mode
for mode in self.modes:
self.mode = mode
ipus_per_replica_mode_str = self._get_managed_attr_mode_name("ipus_per_replica")
# len(matmul_proportion) must equal ipus_per_replica
if isinstance(self._matmul_proportion, list) and len(self._matmul_proportion) != self._ipus_per_replica:
matmul_proportion_mode_str = self._get_managed_attr_mode_name("matmul_proportion")
raise IncompatibleIPUConfigError(
f"{matmul_proportion_mode_str}={self._matmul_proportion} should use the"
f" same number of IPUs as {ipus_per_replica_mode_str}={self._ipus_per_replica}."
)
# layers_per_ipu must have the same length as ipus per replica.
# If wildcards are present in layers_per_ipu, let the call to `model.parallelize`
# handle the validation
if -1 not in self._layers_per_ipu and len(self._layers_per_ipu) != self._ipus_per_replica:
layers_per_ipu_mode_str = self._get_managed_attr_mode_name("layers_per_ipu")
raise IncompatibleIPUConfigError(
f"{layers_per_ipu_mode_str}={self._layers_per_ipu} should use the"
f" same number of IPUs as {ipus_per_replica_mode_str}={self._ipus_per_replica}."
)
# Validate non-transformer layer placement configuration
for layer in ("embedding", "projection"):
mode_layer_splits_per_ipu_str = self._get_managed_attr_mode_name(f"serialized_{layer}_splits_per_ipu")
mode_layer_splits_per_ipu = getattr(self, mode_layer_splits_per_ipu_str)
mode_layer_serialisation_factor_str = self._get_managed_attr_mode_name(f"{layer}_serialization_factor")
mode_layer_serialization_factor = getattr(self, mode_layer_serialisation_factor_str)
# If the user has not provided either the layer_serialization_factor or
# layer_splits_per_ipu, default the layer_serialization_factor to 1
if not (mode_layer_splits_per_ipu or mode_layer_serialization_factor):
setattr(self, mode_layer_serialisation_factor_str, 1)
# If the user provides both options, tell them only one is allowed and what each option is for
if mode_layer_splits_per_ipu and mode_layer_serialization_factor:
raise ValueError(
f"Only one of `{mode_layer_serialisation_factor_str}` and `{mode_layer_splits_per_ipu_str}` should"
f" be used at once. `{mode_layer_serialisation_factor_str}` should be used when you want your"
f" {layer} layer serialised on the same IPU (which IPU depends on the model)."
f" `{mode_layer_splits_per_ipu_str}` should be used when you want your {layer} layer to be split"
" across multiple IPUs of your choice (or to choose which single IPU the layer is serialised on)."
)
# Serialized layer splits per ipu pipeline must have the same pipeline length
# as the number of ipus per replica
if mode_layer_splits_per_ipu and len(mode_layer_splits_per_ipu) != self._ipus_per_replica:
raise ValueError(
f"{mode_layer_splits_per_ipu_str}={mode_layer_splits_per_ipu}"
f" should use the same number of IPUs as {ipus_per_replica_mode_str}={self._ipus_per_replica}."
)
self.mode = old_mode
return self