# This file is MACHINE GENERATED! Do not edit.
# Generated by: tensorflow/python/tools/api/generator/create_python_api.py script.
"""Library for running a computation across multiple devices.

See the guide for overview and examples:
[TensorFlow v1.x](https://www.tensorflow.org/guide/distribute_strategy),
[TensorFlow v2.x](https://www.tensorflow.org/alpha/guide/distribute_strategy).

The intent of this library is that you can write an algorithm in a stylized way
and it will be usable with a variety of different `tf.distribute.Strategy`
implementations. Each descendant will implement a different strategy for
distributing the algorithm across multiple devices/machines.  Furthermore, these
changes can be hidden inside the specific layers and other library classes that
need special treatment to run in a distributed setting, so that most users'
model definition code can run unchanged. The `tf.distribute.Strategy` API works
the same way with eager and graph execution.

*Glossary*

* _Data parallelism_ is where we run multiple copies of the model
  on different slices of the input data. This is in contrast to
  _model parallelism_ where we divide up a single copy of a model
  across multiple devices.
  Note: we only support data parallelism for now, but
  hope to add support for model parallelism in the future.
* A _device_ is a CPU or accelerator (e.g. GPUs, TPUs) on some machine that
  TensorFlow can run operations on (see e.g. `tf.device`). You may have multiple
  devices on a single machine, or be connected to devices on multiple
  machines. Devices used to run computations are called _worker devices_.
  Devices used to store variables are _parameter devices_. For some strategies,
  such as `tf.distribute.MirroredStrategy`, the worker and parameter devices
  will be the same (see mirrored variables below). For others they will be
  different.  For example, `tf.distribute.experimental.CentralStorageStrategy`
  puts the variables on a single device (which may be a worker device or may be
  the CPU), and `tf.distribute.experimental.ParameterServerStrategy` puts the
  variables on separate machines called parameter servers (see below).
* A _replica_ is one copy of the model, running on one slice of the
  input data. Right now each replica is executed on its own
  worker device, but once we add support for model parallelism
  a replica may span multiple worker devices.
* A _host_ is the CPU device on a machine with worker devices, typically
  used for running input pipelines.
* A _worker_ is defined to be the physical machine(s) containing the physical
  devices (e.g. GPUs, TPUs) on which the replicated computation is executed. A
  worker may contain one or more replicas, but contains at least one
  replica. Typically one worker will correspond to one machine, but in the case
  of very large models with model parallelism, one worker may span multiple
  machines. We typically run one input pipeline per worker, feeding all the
  replicas on that worker.
* _Synchronous_, or more commonly _sync_, training is where the updates from
  each replica are aggregated together before updating the model variables. This
  is in contrast to _asynchronous_, or _async_ training, where each replica
  updates the model variables independently. You may also have replicas
  partitioned into groups which are in sync within each group but async between
  groups.
* _Parameter servers_: These are machines that hold a single copy of
  parameters/variables, used by some strategies (right now just
  `tf.distribute.experimental.ParameterServerStrategy`). All replicas that want
  to operate on a variable retrieve it at the beginning of a step and send an
  update to be applied at the end of the step. These can in priniciple support
  either sync or async training, but right now we only have support for async
  training with parameter servers. Compare to
  `tf.distribute.experimental.CentralStorageStrategy`, which puts all variables
  on a single device on the same machine (and does sync training), and
  `tf.distribute.MirroredStrategy`, which mirrors variables to multiple devices
  (see below).
* _Mirrored variables_: These are variables that are copied to multiple
  devices, where we keep the copies in sync by applying the same
  updates to every copy. Normally would only be used with sync training.
* Reductions and all-reduce: A _reduction_ is some method of aggregating
  multiple values into one value, like "sum" or "mean". If a strategy is doing
  sync training, we will perform a reduction on the gradients to a parameter
  from all replicas before applying the update. _All-reduce_ is an algorithm for
  performing a reduction on values from multiple devices and making the result
  available on all of those devices.

Note that we provide a default version of `tf.distribute.Strategy` that is
used when no other strategy is in scope, that provides the same API with
reasonable default behavior.

"""

from __future__ import print_function as _print_function

import sys as _sys

from tensorflow._api.v1.compat.v1.distribute import cluster_resolver
from tensorflow._api.v1.compat.v1.distribute import experimental
from tensorflow.python.distribute.cross_device_ops import CrossDeviceOps
from tensorflow.python.distribute.cross_device_ops import HierarchicalCopyAllReduce
from tensorflow.python.distribute.cross_device_ops import NcclAllReduce
from tensorflow.python.distribute.cross_device_ops import ReductionToOneDevice
from tensorflow.python.distribute.distribute_lib import InputContext
from tensorflow.python.distribute.distribute_lib import InputReplicationMode
from tensorflow.python.distribute.distribute_lib import ReplicaContext
from tensorflow.python.distribute.distribute_lib import StrategyExtendedV1 as StrategyExtended
from tensorflow.python.distribute.distribute_lib import StrategyV1 as Strategy
from tensorflow.python.distribute.distribute_lib import get_loss_reduction
from tensorflow.python.distribute.distribution_strategy_context import experimental_set_strategy
from tensorflow.python.distribute.distribution_strategy_context import get_distribution_strategy as get_strategy
from tensorflow.python.distribute.distribution_strategy_context import get_replica_context
from tensorflow.python.distribute.distribution_strategy_context import has_distribution_strategy as has_strategy
from tensorflow.python.distribute.distribution_strategy_context import in_cross_replica_context
from tensorflow.python.distribute.mirrored_strategy import MirroredStrategyV1 as MirroredStrategy
from tensorflow.python.distribute.one_device_strategy import OneDeviceStrategyV1 as OneDeviceStrategy
from tensorflow.python.distribute.reduce_util import ReduceOp
from tensorflow.python.training.server_lib import Server

del _print_function

from tensorflow.python.util import module_wrapper as _module_wrapper

if not isinstance(_sys.modules[__name__], _module_wrapper.TFModuleWrapper):
  _sys.modules[__name__] = _module_wrapper.TFModuleWrapper(
      _sys.modules[__name__], "compat.v1.distribute", public_apis=None, deprecation=False,
      has_lite=False)
