rlhf_type: grpo

model: Qwen/Qwen2.5-VL-3B-Instruct
dataset: AI-ModelScope/clevr_cogen_a_train#2000
external_plugins: examples/train/grpo/plugin/plugin.py
reward_funcs: [external_r1v_acc, format]
overlong_filter: false
dataset_num_proc: 4
split_dataset_ratio: 0
load_from_cache_file: false

micro_batch_size: 2
global_batch_size: 16
num_generations: 8
steps_per_generation: 4
num_train_epochs: 1
train_iters: 100
logging_steps: 1
seed: 42
max_length: 4096
max_completion_length: 4096
padding_free: false
cross_entropy_loss_fusion: true
gradient_accumulation_fusion: false
lr: 5e-5
lr_warmup_fraction: 0.0
min_lr: 5e-5
attention_backend: flash
temperature: 1.0
beta: 0
epsilon: 0.2
epsilon_high: 0.28
loss_type: grpo
advantage_estimator: grpo
importance_sampling_level: token
log_rollout_offpolicy_metrics: true
system: examples/train/grpo/prompt.txt

use_vllm: true

train:
  gpus: 4
  tuner_type: lora
  lora_rank: 8
  lora_alpha: 32
  tensor_model_parallel_size: 1
  output_dir: megatron_output/ray_grpo_separate

rollout:
  gpus: 4
  vllm_tensor_parallel_size: 1
  vllm_gpu_memory_utilization: 0.8
  vllm_max_model_len: 8192