@misc{ChatHuman2025,
  title = {{ChatHuman}: Language-driven {3D} Human Understanding with Retrieval-Augmented Tool Reasoning},
  abstract = {Numerous methods have been proposed to detect, estimate, and analyze properties of people in images, including the estimation of 3D pose, shape, contact, human-object interaction, emotion, and more. Each of these methods works in isolation instead of synergistically. Here we address this problem and build a language-driven human understanding system that combines and integrates the skills of many different methods. To do so, we fine-tune a Large Language Model (LLM) to select and use a wide variety of existing tools in response to user inputs. In doing so, our ChatHuman is able to combine information from multiple tools to solve problems more accurately than the individual tools themselves and to leverage tool output to improve its ability to reason about humans. The novel features of ChatHuman include leveraging academic publications to guide the application of 3D human-related tools, employing a retrieval augmented generation model to generate in-context-learning examples for handling new tools, and discriminating and integrating tool results to enhance 3D human understanding. Extensive quantitative evaluation shows that ChatHuman outperforms existing models in both tool selection accuracy and performance across multiple 3D human-related tasks. ChatHuman is a step towards consolidating diverse methods for human analysis into a single, powerful, system for reasoning about people in 3D.},
  howpublished = {arXiv},
  month = may,
  year = {2024},
  slug = {chathuman2025},
  author = {Lin, Jing and Feng, Yao and Liu, Weiyang and Black, Michael J.},
  eprint = {2405.04533},
  url = {https://chathuman.github.io/},
  month_numeric = {5}
}