noether.core.distributed

Submodules

Classes

DistributedConfig

Functions

barrier()

get_local_rank()

get_managed_rank()

get_managed_world_size()

get_num_nodes()

get_rank()

get_world_size()

is_data_rank0()

is_distributed()

is_local_rank0()

is_managed()

is_rank0()

set_config(new_config)

all_gather_grad(x[, batch_dim])

all_gather_nograd(x[, batch_dim])

all_gather_nograd_clipped(x[, max_length, batch_dim])

all_reduce_mean_grad(x)

all_reduce_mean_nograd(x)

all_reduce_sum_grad(x)

all_reduce_sum_nograd(x)

reduce_max_grad(x[, dest_rank])

reduce_max_nograd(x[, dest_rank])

reduce_mean_grad(x[, dest_rank])

reduce_mean_nograd(x[, dest_rank])

run(main[, devices, accelerator, master_port])

run_managed(main[, accelerator, devices])

run_unmanaged(main, devices[, accelerator, master_port])

accelerator_to_device(accelerator)

check_single_device_visible(accelerator)

log_device_info(accelerator, device_ids)

parse_devices(accelerator, devices)

Package Contents

class noether.core.distributed.DistributedConfig
static is_managed()
Return type:

bool

get_local_rank()
Return type:

int

get_num_nodes()
Return type:

int

get_managed_world_size()
Return type:

int

get_managed_rank()
Return type:

int

static is_distributed()
Return type:

bool

get_rank()
Return type:

int

get_world_size()
Return type:

int

is_data_rank0()
Return type:

bool

is_rank0()
Return type:

bool

is_local_rank0()
Return type:

bool

barrier()
Return type:

None

noether.core.distributed.barrier()
noether.core.distributed.get_local_rank()
noether.core.distributed.get_managed_rank()
noether.core.distributed.get_managed_world_size()
noether.core.distributed.get_num_nodes()
noether.core.distributed.get_rank()
noether.core.distributed.get_world_size()
noether.core.distributed.is_data_rank0()
noether.core.distributed.is_distributed()
noether.core.distributed.is_local_rank0()
noether.core.distributed.is_managed()
noether.core.distributed.is_rank0()
noether.core.distributed.set_config(new_config)
Parameters:

new_config (DistributedConfig)

noether.core.distributed.all_gather_grad(x, batch_dim=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.all_gather_nograd(x, batch_dim=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.all_gather_nograd_clipped(x, max_length=None, batch_dim=0)
Parameters:
Return type:

torch.Tensor

noether.core.distributed.all_reduce_mean_grad(x)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.all_reduce_mean_nograd(x)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.all_reduce_sum_grad(x)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.all_reduce_sum_nograd(x)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.reduce_max_grad(x, dest_rank=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.reduce_max_nograd(x, dest_rank=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.reduce_mean_grad(x, dest_rank=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.reduce_mean_nograd(x, dest_rank=0)
Parameters:

x (torch.Tensor)

Return type:

torch.Tensor

noether.core.distributed.run(main, devices=None, accelerator='gpu', master_port=None)
noether.core.distributed.run_managed(main, accelerator='gpu', devices=None)
noether.core.distributed.run_unmanaged(main, devices, accelerator='gpu', master_port=None)
Parameters:

  • devices (str | None)

  • accelerator (str)

  • master_port (int | None)

noether.core.distributed.accelerator_to_device(accelerator)
noether.core.distributed.check_single_device_visible(accelerator)
noether.core.distributed.log_device_info(accelerator, device_ids)
noether.core.distributed.parse_devices(accelerator, devices)
Parameters:

  • accelerator (str)

  • devices (str | None)