Repository navigation
Expand file tree
/
Copy pathdpo.yaml
More file actions
executable file
·220 lines (192 loc) · 7.23 KB
/
Copy pathdpo.yaml
File metadata and controls
executable file
·220 lines (192 loc) · 7.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
# DPO Algorithm Configuration
dpo:
max_num_epochs: 1
max_num_steps: 150
val_period: 25
val_batches: 8
val_global_batch_size: 8
val_micro_batch_size: 1
val_at_start: true
seed: 42
reference_policy_kl_penalty: 0.05
preference_average_log_probs: False # whether normalizing log probs according to the sequence length in preference_loss
sft_average_log_probs: ${.preference_average_log_probs} # whether normalizing log probs according to the sequence length in sft_loss
## TODO(@ashors) support other loss functions
#preference_loss: dpo # the preference loss, we support dpo, ipo, rpo_sq, rpo_bwd_kl, rpo_fwd_kl
#gt_reward_scale: 1. # the scale of the rewards in RPO
preference_loss_weight: 1 # the coefficient of the preference loss
sft_loss_weight: 0 # the coefficient of the SFT loss
checkpointing:
enabled: true
checkpoint_dir: "results/dpo"
metric_name: "val:validation-default_loss"
higher_is_better: false
keep_top_k: 3
save_period: 50
checkpoint_must_save_by: null
policy:
model_name: "meta-llama/Llama-3.2-1B-Instruct"
tokenizer:
name: "meta-llama/Llama-3.2-1B-Instruct"
chat_template_kwargs: null # can be used to pass kwargs to the chat template, e.g., enable_thinking=true
# number of preference samples per batch
# each preference sample corresponds to a pair of chosen and rejected responses
# so the actual batch size processed by the model is train_global_batch_size * 2
train_global_batch_size: 128
train_micro_batch_size: 2
## TODO(@ashors) support
#logprob_batch_size: ${policy.train_micro_batch_size}
max_total_sequence_length: 1024
precision: "bfloat16"
offload_optimizer_for_logprob: false
dtensor_cfg:
env_vars:
PYTORCH_CUDA_ALLOC_CONF: "" # Refers to https://docs.pytorch.org/docs/stable/notes/cuda.html#optimizing-memory-usage-with-pytorch-cuda-alloc-conf
enabled: true
cpu_offload: False
sequence_parallel: false
activation_checkpointing: false
tensor_parallel_size: 1
context_parallel_size: 1
custom_parallel_plan: null
clear_cache_every_n_steps: null
dynamic_batching:
enabled: false
sequence_packing:
enabled: false
# makes the training sequence length divisible by the tensor parallel size
# this is useful for sequence parallel training
make_sequence_length_divisible_by: ${policy.dtensor_cfg.tensor_parallel_size}
max_grad_norm: 1.0
optimizer:
name: "torch.optim.AdamW"
kwargs:
lr: 5.0e-6
weight_decay: 0.1
betas: [0.9, 0.98]
eps: 1e-5
# when using Dtensor, we need to set foreach
# and fused to False
foreach: False
fused: False
scheduler:
- name: "torch.optim.lr_scheduler.LinearLR"
kwargs:
start_factor: 0.1
end_factor: 1.0
total_iters: 20
- name: "torch.optim.lr_scheduler.ConstantLR"
kwargs:
factor: 1.0
total_iters: 10000000000
- milestones: [20]
## ignored since enabled=false, but needed for testing purposes
megatron_cfg:
enabled: false
empty_unused_memory_level: 1
activation_checkpointing: false
tensor_model_parallel_size: 2
expert_tensor_parallel_size: 1
expert_model_parallel_size: 1
pipeline_model_parallel_size: 1
context_parallel_size: 1
pipeline_dtype: ${policy.precision}
num_layers_in_first_pipeline_stage: null
num_layers_in_last_pipeline_stage: null
sequence_parallel: true
freeze_moe_router: false
moe_router_dtype: "fp64"
moe_router_load_balancing_type: "aux_loss"
moe_router_bias_update_rate: 1e-3
moe_permute_fusion: false
#gives ~20% training perf speedup with sequence packing
apply_rope_fusion: True
# gives ~25% training perf speedup with sequence packing and apply_rope_fusion
bias_activation_fusion: True
defer_fp32_logits: False
optimizer:
optimizer: "adam"
lr: 5.0e-6 #4.0e-5
min_lr: 5.0e-6 #4.0e-5
weight_decay: 0.1
bf16: true
fp16: false
params_dtype: "float32"
#adam
adam_beta1: 0.9
adam_beta2: 0.98
adam_eps: 1e-8
#sgd
sgd_momentum: 0.9
#distributed optimizer
use_distributed_optimizer: true
use_precision_aware_optimizer: true
clip_grad: ${policy.max_grad_norm}
# optimizer cpu offload
optimizer_cpu_offload: false
optimizer_offload_fraction: 0.0
scheduler:
start_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay}
end_weight_decay: ${policy.megatron_cfg.optimizer.weight_decay}
weight_decay_incr_style: "constant"
lr_decay_style: "constant"
lr_warmup_iters: 1
lr_warmup_init: 0.00000001
distributed_data_parallel_config:
grad_reduce_in_fp32: false
overlap_grad_reduce: true
overlap_param_gather: true
data_parallel_sharding_strategy: "optim_grads_params"
use_custom_fsdp: false
data:
max_input_seq_length: ${policy.max_total_sequence_length}
shuffle: true
num_workers: 1
dataset_name: HelpSteer3
# You can use custom preference datasets for training and validation. For example:
# 1. PreferenceDataset
# data:
# dataset_name: PreferenceDataset
# train_data_path: <PathToTrainingDataset> # e.g., /path/to/local/dataset.jsonl or hf_org/hf_dataset_name (HuggingFace)
# val_data_paths:
# <NameOfValidationDataset1>: <PathToValidationDataset1>
# ...
# train_split: <TrainSplit>, default is None # used for HuggingFace datasets
# val_split: <ValSplit>, default is None # used for HuggingFace datasets
# 2. BinaryPreferenceDataset
# data:
# dataset_name: BinaryPreferenceDataset
# train_data_path: <PathToTrainingDataset> # e.g., /path/to/local/dataset.jsonl or hf_org/hf_dataset_name (HuggingFace)
# val_data_path: <PathToValidationDataset>
# prompt_key: <PromptKey>, default is "prompt"
# chosen_key: <ChosenKey>, default is "chosen"
# rejected_key: <RejectedKey>, default is "rejected"
# train_split: <TrainSplit>, default is None # used for HuggingFace datasets
# val_split: <ValSplit>, default is None # used for HuggingFace datasets
# See https://github.com/NVIDIA-NeMo/RL/blob/main/docs/guides/dpo.md#datasets for more details.
# If you are doing checkpointing, `metric_name` should reflect the metric and validation set to be tracked. For example:
# checkpointing:
# metric_name: "val:validation-<NameOfValidationDataset1>_loss"
# ...
logger:
log_dir: "logs" # Base directory for all logs
wandb_enabled: false # Make sure you do a ``wandb login [Your API key]'' before running
tensorboard_enabled: false
mlflow_enabled: false # Disable MLflow logging
swanlab_enabled: false # Disable SwanLab logging
monitor_gpus: true # If true, will monitor GPU usage and log to wandb and/or tensorboard
num_val_samples_to_print: 0 # Number of validation samples to pretty print on terminal
wandb:
project: "dpo-dev"
name: "dpo"
tensorboard:
log_dir: "tb_logs-dpo-dev"
mlflow:
experiment_name: "dpo-dev"
run_name: "dpo"
gpu_monitoring:
collection_interval: 10 # How often to collect GPU usage metrics (in seconds)
flush_interval: 10 # How often to flush GPU usage metrics to the loggers (in seconds)
cluster:
gpus_per_node: 1
num_nodes: 1