@article{relu2winsdiscoveringefficientactivation,
  title = {ReLU$^2$ Wins: Discovering Efficient Activation Functions for Sparse LLMs},
  author = {Zhengyan Zhang and Yixin Song and Guanghui Yu and Xu Han and Yankai Lin and Chaojun Xiao and Chenyang Song and Zhiyuan Liu and Zeyu Mi and Maosong Sun},
  year = {2024},
  eprint = {2402.03804},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.03804v1},
}