> ## Documentation Index > Fetch the complete documentation index at: https://docs.trainy.ai/llms.txt > Use this file to discover all available pages before exploring further. # Ray/VeRL - LLM Reinforcement Learning > Production-ready RL training library for large language models (LLMs) by VeRL. This example shows how to train multinode a large language model via Reinforcement Learning using [**verl: Volcano Engine Reinforcement Learning for LLMs.**](https://verl.readthedocs.io/en/latest/) Define the following `verl.yaml` ``` # Multi-node distributed training with Verl (Volcano Engine Reinforcement Learning) framework. # # Verl is a flexible and efficient reinforcement learning framework designed for # training large language models with RLHF (Reinforcement Learning from Human Feedback). # This example demonstrates multi-node training using PPO on the GSM8K dataset. # # Prerequisites: # - Access to Hugging Face models (Qwen/Qwen2.5-0.5B-Instruct in this example) # # Usage: # # Launch a 2-node training cluster: # $ konduktor launch examples/verl/multinode.yaml # # # Stream logs: # $ konduktor logs verl-cluster-* # # # Cleanup: # $ konduktor down verl-cluster-* # Known Issues: Need to set GLOO_SOCKET_IFNAME to primary network IF which changes # per platform name: verl-multinode-training resources: accelerators: H200:8 image_id: verlai/verl:app-verl0.5-transformers4.55.4-vllm0.10.0-mcore0.13.0-te2.2 cpus: 180 memory: 1000 labels: kueue.x-k8s.io/queue-name: user-queue maxRunDurationSeconds: "320000" num_nodes: 2 # Number of nodes for distributed training # Environment variables envs: HF_HUB_ENABLE_HF_TRANSFER: "1" TORCH_NCCL_AVOID_RECORD_STREAMS: "1" # Optional: Add your W&B API key for experiment tracking WANDB_API_KEY: # Training configuration MODEL_NAME: Qwen/Qwen2.5-0.5B-Instruct TOTAL_EPOCHS: 3 ACTOR_LR: 1e-6 CRITIC_LR: 1e-5 run: | set -x export PIP_CONSTRAINT="" mkdir -p $HOME/checkpoints apt update -y apt install -y iputils-ping git clone https://github.com/volcengine/verl.git cd verl # Install Verl and its dependencies (skip Megatron for this example) #USE_MEGATRON=0 bash scripts/install_vllm_sglang_mcore.sh pip install --no-deps -e . #pip install "ray[default]" # For Ray dashboard # Set up distributed training environment echo "Head IP: $MASTER_ADDR" echo "Number of nodes: $NUM_NODES" # Create custom runtime environment configuration cat > runtime_env_custom.yaml </dev/null 2>&1 } if [ "$RANK" == "0" ]; then # Head node: prepare data, download model, start Ray head, and submit training job echo "Setting up head node..." # Install additional dependencies for data processing pip install datasets transformers # Prepare GSM8K dataset python3 examples/data_preprocess/gsm8k.py --local_dir ~/data/gsm8k # Download model to cache python3 -c "import transformers; transformers.pipeline('text-generation', model='Qwen/Qwen2.5-0.5B-Instruct')" # Start Ray head node if not already running if ! is_ray_alive "$MASTER_ADDR"; then echo "Starting Ray head node..." ray start --head --node-ip-address="$MASTER_ADDR" \ --port $HEAD_PORT --dashboard-port $DASH_PORT \ --dashboard-host=0.0.0.0 \ --dashboard-agent-listen-port=52366 \ --disable-usage-stats \ --num-gpus=$NUM_GPUS_PER_NODE sleep 10 else echo "Ray is already running at $MASTER_ADDR:$HEAD_PORT, reusing existing instance" ray status --address="$MASTER_ADDR:$HEAD_PORT" fi ray status ibdev2netdev # Submit the training job to Ray export RAY_ADDRESS="http://localhost:$DASH_PORT" echo "Submitting training job to Ray cluster..." HYDRA_FULL_ERROR=1 ray job submit --address="$RAY_ADDRESS" --working-dir=. \ --runtime-env=runtime_env_custom.yaml \ -- python3 -m verl.trainer.main_ppo \ data.train_files=$HOME/data/gsm8k/train.parquet \ data.val_files=$HOME/data/gsm8k/test.parquet \ data.train_batch_size=128 \ data.max_prompt_length=512 \ data.max_response_length=256 \ actor_rollout_ref.model.path=$MODEL_NAME \ actor_rollout_ref.actor.optim.lr=$ACTOR_LR \ actor_rollout_ref.actor.ppo_mini_batch_size=64 \ actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ actor_rollout_ref.rollout.name=vllm \ actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=8 \ actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \ actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ critic.optim.lr=$CRITIC_LR \ critic.model.path=$MODEL_NAME \ critic.ppo_micro_batch_size_per_gpu=4 \ critic.ppo_mini_batch_size=64 \ algorithm.kl_ctrl.kl_coef=0.001 \ trainer.project_name=ppo_training \ trainer.experiment_name=qwen-2.5-0.5B \ trainer.val_before_train=False \ trainer.n_gpus_per_node=$NUM_GPUS_PER_NODE \ trainer.nnodes=$NUM_NODES \ trainer.default_local_dir=$HOME/checkpoints \ trainer.save_freq=10 \ trainer.test_freq=10 \ trainer.total_epochs=$TOTAL_EPOCHS \ trainer.logger=['console'] \ trainer.resume_mode=auto 2>&1 | tee verl_training.log # To enable W&B logging: # 1. Set WANDB_API_KEY in envs or pass via --secret WANDB_API_KEY # 2. Change trainer.logger to: trainer.logger=['console', 'wandb'] # 3. Add: trainer.wandb_project='verl-rlhf' exit 1 # this forces the other workers to shutdown else # Worker nodes: connect to Ray head echo "Setting up worker node..." echo "Head IP: $MASTER_ADDR" echo "HEAD_PORT: $HEAD_PORT" echo "NUM_GPUS: $NUM_GPUS_PER_NODE" # Get this worker's IP address worker_ip=$(hostname -I | awk '{print $1}') echo "Worker IP: $worker_ip" echo "Checking if worker $worker_ip is already in Ray cluster at $MASTER_ADDR:$HEAD_PORT" if ray list nodes --address=$MASTER_ADDR:$HEAD_PORT 2>/dev/null | grep -q "$worker_ip"; then echo "Worker $worker_ip already connected to Ray cluster" ray status --address=$MASTER_ADDR:$HEAD_PORT else echo "Worker not connected, waiting for head node to start" sleep 20 echo "Starting Ray worker" ray start --address $MASTER_ADDR:$HEAD_PORT --disable-usage-stats --num-gpus=$NUM_GPUS_PER_NODE echo "Ray start exit code: $?" # Verify connection after starting sleep 5 ray status --address=$MASTER_ADDR:$HEAD_PORT fi while ping -c 1 ${MASTER_ADDR}; do sleep 5 done fi ``` and launch with ``` konduktor launch verl.yaml ``` While the job is running you should see a sample output like in the following ``` (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) [prompt] system (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) You are Qwen, created by Alibaba Cloud. You are a helpful assistant. (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) user (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) Janet’s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market? Let's think step by step and output the final answer after "####". (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) assistant (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) [response] First, we calculate the number of eggs Janet's ducks lay in a day. Since she lays 16 eggs per day, the number of eggs she eats in a day is 16 - 3 - 4 = 10 eggs. This means she sells 10 eggs daily at the farmers' market. Since each egg sells for $2, the daily earnings are 10 * 2 = $20. Therefore, the amount of money Janet makes every day at the farmers' market is $20. (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) #### 20 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) #### 20 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) [ground_truth] 18 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) [score] 0.0 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) len reward_extra_infos_dict['reward']: 1319 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) local_global_step_folder: /root/checkpoints/global_step_174 (job_name=verl-multinode-training-7550 worker_id=0) (WorkerDict pid=1498, ip=172.17.1.94) INFO:2025-08-25 23:17:09,357:[Rank 0] Saved model to /root/checkpoints/global_step_174/actor/model_world_size_16_rank_0.pt (job_name=verl-multinode-training-7550 worker_id=0) (WorkerDict pid=1499, ip=172.17.1.94) INFO:2025-08-25 23:17:09,336:[Rank 1] Saved model to /root/checkpoints/global_step_174/actor/model_world_size_16_rank_1.pt (job_name=verl-multinode-training-7550 worker_id=0) (WorkerDict pid=24109) INFO:2025-08-25 23:17:09,614:[Rank 10] Saved optim to /root/checkpoints/global_step_174/actor/optim_world_size_16_rank_10.pt (job_name=verl-multinode-training-7550 worker_id=0) (WorkerDict pid=24109) INFO:2025-08-25 23:17:09,615:[Rank 10] Saved extra_state to /root/checkpoints/global_step_174/actor/extra_state_world_size_16_rank_10.pt (job_name=verl-multinode-training-7550 worker_id=0) (WorkerDict pid=1498, ip=172.17.1.94) INFO:2025-08-25 23:17:10,020:[Rank 0] Saved model config and tokenizer class to /root/checkpoints/global_step_174/actor/huggingface (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) step:174 - global_seqlen/min:1756 - global_seqlen/max:2241 - global_seqlen/minmax_diff:485 - global_seqlen/balanced_min:1974 - global_seqlen/balanced_max:1977 - global_seqlen/mean:1976.0 - actor/entropy:0.10149244964122772 - critic/vf_loss:0.051519379019737244 - critic/vf_clipfrac:0.0 - critic/vpred_mean:0.8081025183200836 - critic/grad_norm:36.863529205322266 - perf/mfu/critic:0.01831457949805057 - critic/lr:1e-05 - actor/pg_loss:-0.5662342011928558 - actor/pg_clipfrac:0.0051369862630963326 - actor/ppo_kl:-0.0011725000804290175 - actor/pg_clipfrac_lower:0.0 - actor/grad_norm:4.237640619277954 - perf/mfu/actor:0.0185783582944669 - perf/max_memory_allocated_gb:56.59689235687256 - perf/max_memory_reserved_gb:59.7578125 - perf/cpu_memory_used_gb:58.61131286621094 - actor/lr:1e-06 - val-core/openai/gsm8k/reward/mean@1:0.4890068233510235 - training/global_step:174 - training/epoch:2 - critic/score/mean:0.609375 - critic/score/max:1.0 - critic/score/min:0.0 - critic/rewards/mean:0.609375 - critic/rewards/max:1.0 - critic/rewards/min:0.0 - critic/advantages/mean:1.1653965792390863e-08 - critic/advantages/max:2.7394912242889404 - critic/advantages/min:-2.5047049522399902 - critic/returns/mean:0.5375215411186218 - critic/returns/max:1.0 - critic/returns/min:0.0 - critic/values/mean:0.73046875 - critic/values/max:1.2421875 - critic/values/min:0.01556396484375 - critic/vf_explained_var:0.3024832606315613 - response_length/mean:145.125 - response_length/max:256.0 - response_length/min:75.0 - response_length/clip_ratio:0.0546875 - response_length_non_aborted/mean:145.125 - response_length_non_aborted/max:256.0 - response_length_non_aborted/min:75.0 - response_length_non_aborted/clip_ratio:0.0546875 - response/aborted_ratio:0.0 - prompt_length/mean:101.875 - prompt_length/max:169.0 - prompt_length/min:73.0 - prompt_length/clip_ratio:0.0 - timing_s/start_profile:5.1025766879320145e-05 - timing_s/generate_sequences:0.6771697402000427 - timing_s/reshard:0.5008471012115479 - timing_s/generation_timing/max:0.7759467363357544 - timing_s/generation_timing/min:0.49384644627571106 - timing_s/generation_timing/topk_ratio:0.125 - timing_s/gen:1.8636266626417637 - timing_s/reward:0.017636850010603666 - timing_s/old_log_prob:0.2443925621919334 - timing_s/values:0.14951528888195753 - timing_s/adv:0.01702857529744506 - timing_s/update_critic:0.44504289515316486 - timing_s/update_actor:0.43918753834441304 - timing_s/step:3.178868151269853 - timing_s/testing:3.8995088669471443 - timing_s/save_checkpoint:1.738096852786839 - timing_s/stop_profile:5.9116631746292114e-05 - timing_per_token_ms/update_actor:0.01389130624824181 - timing_per_token_ms/adv:0.0005386062530821439 - timing_per_token_ms/update_critic:0.014076508576453848 - timing_per_token_ms/gen:0.10032443274341966 - timing_per_token_ms/values:0.004729102001580134 - perf/total_num_tokens:31616 - perf/time_per_step:3.178868151269853 - perf/throughput:621.6048939339158 (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) ("Final validation metrics: {'val-core/openai/gsm8k/reward/mean@1': " (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) '0.4890068233510235}') (job_name=verl-multinode-training-7550 worker_id=0) (TaskRunner pid=14513) (job_name=verl-multinode-training-7550 worker_id=0) Training Progress: 100%|██████████| 174/174 [14:01<00:00, 5.33s/it] ```