@inproceedings{liu-etal-2024-forgetting,
title = "Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models",
author = "Liu, Xinyu and
Zhao, Runsong and
Huang, Pengcheng and
Xiao, Chunyang and
Li, Bei and
Wang, Jingang and
Xiao, Tong and
Zhu, JingBo",
editor = "Al-Onaizan, Yaser and
Bansal, Mohit and
Chen, Yun-Nung",
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing",
month = nov,
year = "2024",
address = "Miami, Florida, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.emnlp-main.269/",
doi = "10.18653/v1/2024.emnlp-main.269",
pages = "4667--4682",
abstract = "Numerous recent works target to extend effective context length for language models and various methods, tasks and benchmarks exist to measure model`s effective memory length. However, through thorough investigations, we find limitations for currently existing evaluations on model`s memory. We provide an extensive survey for limitations in this work and propose a new method called forgetting curve to measure the memorization capability of long-context models. We show that forgetting curve has the advantage of being robust to the tested corpus and the experimental settings, of not relying on prompt and can be applied to any model size. We apply our forgetting curve to a large variety of models involving both transformer and RNN/SSM based architectures. Our measurement provides empirical evidence for the effectiveness of transformer extension techniques while raises questions for the effective length of RNN/SSM based models. We also examine the difference between our measurement and existing benchmarks as well as popular metrics for various models."
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="liu-etal-2024-forgetting">
<titleInfo>
<title>Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Xinyu</namePart>
<namePart type="family">Liu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Runsong</namePart>
<namePart type="family">Zhao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pengcheng</namePart>
<namePart type="family">Huang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Chunyang</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Bei</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jingang</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Tong</namePart>
<namePart type="family">Xiao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">JingBo</namePart>
<namePart type="family">Zhu</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2024-11</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing</title>
</titleInfo>
<name type="personal">
<namePart type="given">Yaser</namePart>
<namePart type="family">Al-Onaizan</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mohit</namePart>
<namePart type="family">Bansal</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Yun-Nung</namePart>
<namePart type="family">Chen</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Miami, Florida, USA</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Numerous recent works target to extend effective context length for language models and various methods, tasks and benchmarks exist to measure model‘s effective memory length. However, through thorough investigations, we find limitations for currently existing evaluations on model‘s memory. We provide an extensive survey for limitations in this work and propose a new method called forgetting curve to measure the memorization capability of long-context models. We show that forgetting curve has the advantage of being robust to the tested corpus and the experimental settings, of not relying on prompt and can be applied to any model size. We apply our forgetting curve to a large variety of models involving both transformer and RNN/SSM based architectures. Our measurement provides empirical evidence for the effectiveness of transformer extension techniques while raises questions for the effective length of RNN/SSM based models. We also examine the difference between our measurement and existing benchmarks as well as popular metrics for various models.</abstract>
<identifier type="citekey">liu-etal-2024-forgetting</identifier>
<identifier type="doi">10.18653/v1/2024.emnlp-main.269</identifier>
<location>
<url>https://aclanthology.org/2024.emnlp-main.269/</url>
</location>
<part>
<date>2024-11</date>
<extent unit="page">
<start>4667</start>
<end>4682</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models
%A Liu, Xinyu
%A Zhao, Runsong
%A Huang, Pengcheng
%A Xiao, Chunyang
%A Li, Bei
%A Wang, Jingang
%A Xiao, Tong
%A Zhu, JingBo
%Y Al-Onaizan, Yaser
%Y Bansal, Mohit
%Y Chen, Yun-Nung
%S Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing
%D 2024
%8 November
%I Association for Computational Linguistics
%C Miami, Florida, USA
%F liu-etal-2024-forgetting
%X Numerous recent works target to extend effective context length for language models and various methods, tasks and benchmarks exist to measure model‘s effective memory length. However, through thorough investigations, we find limitations for currently existing evaluations on model‘s memory. We provide an extensive survey for limitations in this work and propose a new method called forgetting curve to measure the memorization capability of long-context models. We show that forgetting curve has the advantage of being robust to the tested corpus and the experimental settings, of not relying on prompt and can be applied to any model size. We apply our forgetting curve to a large variety of models involving both transformer and RNN/SSM based architectures. Our measurement provides empirical evidence for the effectiveness of transformer extension techniques while raises questions for the effective length of RNN/SSM based models. We also examine the difference between our measurement and existing benchmarks as well as popular metrics for various models.
%R 10.18653/v1/2024.emnlp-main.269
%U https://aclanthology.org/2024.emnlp-main.269/
%U https://doi.org/10.18653/v1/2024.emnlp-main.269
%P 4667-4682
Markdown (Informal)
[Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models](https://aclanthology.org/2024.emnlp-main.269/) (Liu et al., EMNLP 2024)
ACL
- Xinyu Liu, Runsong Zhao, Pengcheng Huang, Chunyang Xiao, Bei Li, Jingang Wang, Tong Xiao, and JingBo Zhu. 2024. Forgetting Curve: A Reliable Method for Evaluating Memorization Capability for Long-Context Models. In Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pages 4667–4682, Miami, Florida, USA. Association for Computational Linguistics.