ML-Agents案例之食物收集者

本文主要是介绍ML-Agents案例之食物收集者，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

本案例源自ML-Agents官方的示例，Github地址：https://github.com/Unity-Technologies/ml-agents，本文是详细的配套讲解。

本文基于我前面发的两篇文章，需要对ML-Agents有一定的了解，详情请见：Unity强化学习之ML-Agents的使用、ML-Agents命令及配置大全。

我前面的相关文章有：

ML-Agents案例之Crawler

ML-Agents案例之推箱子游戏

ML-Agents案例之跳墙游戏

环境说明

在这里插入图片描述

环境中存在有多个智能体，他们的任务是收集尽可能多的绿色食物球，并避免碰到红色球：碰到绿色球奖励+1，碰到红色球-1，此外，智能体之间可以通过发射射线冻结其他智能体，以让自己吃到更多的食物球，达到增加自己分数的目的。

观察空间：使用了网格传感器（Grid Sensor），这个传感器具体说明参考ML-Agents案例之推箱子游戏的多人模式。

在本案例中传感器直接挂载在智能体本身，z轴（前后方向）网格数为40，x轴（左右方向）网格数为40，y轴（上下方向）网格数为1，检测的标签有，食物，坏食物，其他智能体，被冻结的智能体，墙壁，加上什么都没观测到，维度为40 * 40 * 6 = 960个观察维度。

在这里插入图片描述

动作空间：3个连续动作输出，对应于前后运动，左右运动和旋转。1个离散输出，对应于是否发生激光。

代码分析

食物生成

控制食物生成的脚本，挂载在空物体上，这个脚本不会自己运作，需要在智能体挂载的脚本上调用。

using UnityEngine;
using Unity.MLAgentsExamples;public class FoodCollectorArea : Area
{public GameObject food;public GameObject badFood;public int numFood;public int numBadFood;public bool respawnFood;public float range;// 生成食物void CreateFood(int num, GameObject type){for (int i = 0; i < num; i++){GameObject f = Instantiate(type, new Vector3(Random.Range(-range, range), 1f,Random.Range(-range, range)) + transform.position,Quaternion.Euler(new Vector3(0f, Random.Range(0f, 360f), 90f)));f.GetComponent<FoodLogic>().respawn = respawnFood;f.GetComponent<FoodLogic>().myArea = this;}}// 重置区域，随机化智能体们的位置，并生成两种食物public void ResetFoodArea(GameObject[] agents){foreach (GameObject agent in agents){if (agent.transform.parent == gameObject.transform){agent.transform.position = new Vector3(Random.Range(-range, range), 2f,Random.Range(-range, range))+ transform.position;agent.transform.rotation = Quaternion.Euler(new Vector3(0f, Random.Range(0, 360)));}}CreateFood(numFood, food);CreateFood(numBadFood, badFood);}public override void ResetArea(){}
}

食物被吃掉的处理

挂载在食物上的脚本，同样需要调用才能起作用：

using UnityEngine;public class FoodLogic : MonoBehaviour
{public bool respawn;public FoodCollectorArea myArea;// 被吃掉后的两种选择，一个是重新随机位置，一个是直接销毁public void OnEaten(){if (respawn){transform.position = new Vector3(Random.Range(-myArea.range, myArea.range),3f,Random.Range(-myArea.range, myArea.range)) + myArea.transform.position;}else{Destroy(gameObject);}}
}

智能体主文件FoodCollectorAgent.cs：

初始化：

public override void Initialize()
{m_AgentRb = GetComponent<Rigidbody>();m_MyArea = area.GetComponent<FoodCollectorArea>();m_FoodCollecterSettings = FindObjectOfType<FoodCollectorSettings>();// 从配置文件中获取参数m_ResetParams = Academy.Instance.EnvironmentParameters;// 设置参数SetResetParameters();
}
// 激光长度
public void SetLaserLengths()
{m_LaserLength = m_ResetParams.GetWithDefault("laser_length", 1.0f);
}
// 设置智能体体积大小
public void SetAgentScale()
{float agentScale = m_ResetParams.GetWithDefault("agent_scale", 1.0f);gameObject.transform.localScale = new Vector3(agentScale, agentScale, agentScale);
}public void SetResetParameters()
{SetLaserLengths();SetAgentScale();
}

观察输入（可以通过设置useVectorObs和useVectorFrozenFlag）来配置是否输入，在案例中没有选择开启这两项输入：

public override void CollectObservations(VectorSensor sensor)
{if (useVectorObs){var localVelocity = transform.InverseTransformDirection(m_AgentRb.velocity);// 输入水平方向的两个速度sensor.AddObservation(localVelocity.x);sensor.AddObservation(localVelocity.z);// 输入是否冻结和是否发射sensor.AddObservation(m_Frozen);sensor.AddObservation(m_Shoot);}else if (useVectorFrozenFlag){// 输入是否冻结和是否发射sensor.AddObservation(m_Frozen);}
}

动作输出：

// 执行输出的主函数，里面的内容都封装到MoveAgent了
public override void OnActionReceived(ActionBuffers actionBuffers)
{MoveAgent(actionBuffers);
}public void MoveAgent(ActionBuffers actionBuffers)
{m_Shoot = false;// 超过冻结时间，解冻if (Time.time > m_FrozenTime + 4f && m_Frozen){Unfreeze();}// 超过被毒时间，解除毒素；超过满意时间，变为正常状态if (Time.time > m_EffectTime + 0.5f){if (m_Poisoned){Unpoison();}if (m_Satiated){Unsatiate();}}var dirToGo = Vector3.zero;var rotateDir = Vector3.zero;var continuousActions = actionBuffers.ContinuousActions;var discreteActions = actionBuffers.DiscreteActions;// 没有冻结的时候才能执行动作if (!m_Frozen){// 获取三个连续输出var forward = Mathf.Clamp(continuousActions[0], -1f, 1f);var right = Mathf.Clamp(continuousActions[1], -1f, 1f);var rotate = Mathf.Clamp(continuousActions[2], -1f, 1f);dirToGo = transform.forward * forward;dirToGo += transform.right * right;rotateDir = -transform.up * rotate;// 获取一个离散输出var shootCommand = discreteActions[0] > 0;if (shootCommand){// 射击时减速m_Shoot = true;dirToGo *= 0.5f;m_AgentRb.velocity *= 0.75f;}// 执行移动m_AgentRb.AddForce(dirToGo * moveSpeed, ForceMode.VelocityChange);transform.Rotate(rotateDir, Time.fixedDeltaTime * turnSpeed);}// 超过一定速度需要减速if (m_AgentRb.velocity.sqrMagnitude > 25f) // slow it down{m_AgentRb.velocity *= 0.95f;}// 执行射击逻辑if (m_Shoot){var myTransform = transform;myLaser.transform.localScale = new Vector3(1f, 1f, m_LaserLength);var rayDir = 25.0f * myTransform.forward;// 绘制射线，参数为起始位置，长度向量，颜色，持续时间，能否被遮挡Debug.DrawRay(myTransform.position, rayDir, Color.red, 0f, true);RaycastHit hit;// 发出球状射线，参数为起始位置，球半径，距离，碰撞到的物体，距离if (Physics.SphereCast(transform.position, 2f, rayDir, out hit, 25f)){// 射线碰到其他智能体，其他智能体会被冻结if (hit.collider.gameObject.CompareTag("agent")){hit.collider.gameObject.GetComponent<FoodCollectorAgent>().Freeze();}}}else{myLaser.transform.localScale = new Vector3(0f, 0f, 0f);}
}
// 被冻结的逻辑，修改标签，计算时间，替换材质
void Freeze()
{gameObject.tag = "frozenAgent";m_Frozen = true;m_FrozenTime = Time.time;gameObject.GetComponentInChildren<Renderer>().material = frozenMaterial;
}
// 解冻的逻辑
void Unfreeze()
{m_Frozen = false;gameObject.tag = "agent";gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}// 吃到坏食物的状态
void Poison()
{m_Poisoned = true;m_EffectTime = Time.time;gameObject.GetComponentInChildren<Renderer>().material = badMaterial;
}
// 解毒
void Unpoison()
{m_Poisoned = false;gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}
// 吃到好食物的状态
void Satiate()
{m_Satiated = true;m_EffectTime = Time.time;gameObject.GetComponentInChildren<Renderer>().material = goodMaterial;
}
// 好状态消失
void Unsatiate()
{m_Satiated = false;gameObject.GetComponentInChildren<Renderer>().material = normalMaterial;
}

和食物碰撞的逻辑：

void OnCollisionEnter(Collision collision)
{// 吃到好食物，奖励一分，进入满意状态if (collision.gameObject.CompareTag("food")){Satiate();collision.gameObject.GetComponent<FoodLogic>().OnEaten();AddReward(1f);if (contribute){m_FoodCollecterSettings.totalScore += 1;}}// 吃到坏食物，扣除1分，进入中毒状态if (collision.gameObject.CompareTag("badFood")){Poison();collision.gameObject.GetComponent<FoodLogic>().OnEaten();AddReward(-1f);if (contribute){m_FoodCollecterSettings.totalScore -= 1;}}
}

一轮游戏开始执行的逻辑：

public override void OnEpisodeBegin()
{// 消除所有异常状态Unfreeze();Unpoison();Unsatiate();m_Shoot = false;// 速度位置都归零m_AgentRb.velocity = Vector3.zero;myLaser.transform.localScale = new Vector3(0f, 0f, 0f);transform.position = new Vector3(Random.Range(-m_MyArea.range, m_MyArea.range),2f, Random.Range(-m_MyArea.range, m_MyArea.range))+ area.transform.position;transform.rotation = Quaternion.Euler(new Vector3(0f, Random.Range(0, 360)));// 重新设置参数，包括激光长短，智能体大小SetResetParameters();
}

可以通过以下代码自己控制智能体的输出：

 public override void Heuristic(in ActionBuffers actionsOut){var continuousActionsOut = actionsOut.ContinuousActions;if (Input.GetKey(KeyCode.D)){continuousActionsOut[2] = 1;}if (Input.GetKey(KeyCode.W)){continuousActionsOut[0] = 1;}if (Input.GetKey(KeyCode.A)){continuousActionsOut[2] = -1;}if (Input.GetKey(KeyCode.S)){continuousActionsOut[0] = -1;}var discreteActionsOut = actionsOut.DiscreteActions;discreteActionsOut[0] = Input.GetKey(KeyCode.Space) ? 1 : 0;}

配置文件

配置文件是普通的PPO和SAC配置：

behaviors:GridFoodCollector:trainer_type: ppohyperparameters:batch_size: 1024buffer_size: 10240learning_rate: 0.0003beta: 0.005epsilon: 0.2lambd: 0.95num_epoch: 3learning_rate_schedule: linearnetwork_settings:normalize: falsehidden_units: 256num_layers: 1vis_encode_type: simplereward_signals:extrinsic:gamma: 0.99strength: 1.0keep_checkpoints: 5max_steps: 2000000time_horizon: 64summary_freq: 10000

behaviors:GridFoodCollector:trainer_type: sachyperparameters:learning_rate: 0.0003learning_rate_schedule: constantbatch_size: 256buffer_size: 2048buffer_init_steps: 0tau: 0.005steps_per_update: 10.0save_replay_buffer: falseinit_entcoef: 0.05reward_signal_steps_per_update: 10.0network_settings:normalize: falsehidden_units: 256num_layers: 1vis_encode_type: simplereward_signals:extrinsic:gamma: 0.99strength: 1.0keep_checkpoints: 5max_steps: 2000000time_horizon: 64summary_freq: 60000threaded: false