{ "ok": true, "doi": "10.46243/jst.2021.v6.i06.pp139-149", "events": [ { "seq": 1, "kind": "register", "at": "2026-09-29 22:00:41", "by": "Administrator (admin)", "by_kind": "user", "note": "registered at Crossref; record read from api.crossref.org", "changes": [ { "path": "abstract.lang", "from": null, "to": "en" }, { "path": "abstract.value", "from": null, "to": "Benchmarks have been used in computer science to compare the performance of computer systems, information retrieval algorithms, databases, and many other technologies. The creation and widespread use of a benchmark within a research area is frequently accompanied by rapid technical progress and community building. These observations have led us to formulate a theory of benchmarking within scientific disciplines. Based on this theory, we challenge software engineering research to become more scientific and cohesive by working as a community to define benchmarks. In support of this challenge, we present a case study of the reverse engineering community, where we have successfully used benchmarks to advance the state of research." }, { "path": "agents.0.name.family", "from": null, "to": "Kumar" }, { "path": "agents.0.name.given", "from": null, "to": "K Vijay" }, { "path": "agents.0.role", "from": null, "to": "author" }, { "path": "agents.0.sequence", "from": null, "to": "first" }, { "path": "agents.1.name.family", "from": null, "to": "Naga Pawan" }, { "path": "agents.1.name.given", "from": null, "to": "Dr.Y V R" }, { "path": "agents.1.role", "from": null, "to": "author" }, { "path": "agents.1.sequence", "from": null, "to": "additional" }, { "path": "agents.2.name.org", "from": null, "to": "Longman Publishers" }, { "path": "agents.2.role", "from": null, "to": "publisher" }, { "path": "characters.0", "from": null, "to": "Language" }, { "path": "container.identifiers.0.medium", "from": null, "to": "electronic" }, { "path": "container.identifiers.0.type", "from": null, "to": "ISSN" }, { "path": "container.identifiers.0.value", "from": null, "to": "2456-5660" }, { "path": "container.issue", "from": null, "to": "06" }, { "path": "container.pages.first", "from": null, "to": "139" }, { "path": "container.pages.last", "from": null, "to": "149" }, { "path": "container.titles.0.type", "from": null, "to": "PrincipalTitle" }, { "path": "container.titles.0.value", "from": null, "to": "Journal of Science & Technology" }, { "path": "container.type", "from": null, "to": "Journal" }, { "path": "container.volume", "from": null, "to": "06" }, { "path": "dates.date_type", "from": null, "to": "PublicationDate" }, { "path": "dates.online", "from": null, "to": "2021" }, { "path": "dates.published", "from": null, "to": "2021" }, { "path": "doi", "from": null, "to": "10.46243/jst.2021.v6.i06.pp139-149" }, { "path": "format", "from": null, "to": "smartscholars-doi-metadata/1.0" }, { "path": "identifiers.0.type", "from": null, "to": "DOI" }, { "path": "identifiers.0.value", "from": null, "to": "10.46243/jst.2021.v6.i06.pp139-149" }, { "path": "language", "from": null, "to": "en" }, { "path": "license.applies_to", "from": null, "to": "vor" }, { "path": "license.start", "from": null, "to": "2021-01-01" }, { "path": "license.url", "from": null, "to": "https://creativecommons.org/licenses/by/4.0/" }, { "path": "links.0.primary", "from": null, "to": true }, { "path": "links.0.return_type", "from": null, "to": "text/html" }, { "path": "links.0.url", "from": null, "to": "https://www.jst.org.in/index.php/pub/article/view/532" }, { "path": "links.1.purpose", "from": null, "to": "text-mining" }, { "path": "links.1.return_type", "from": null, "to": "application/pdf" }, { "path": "links.1.url", "from": null, "to": "https://www.jst.org.in/index.php/pub/article/download/532/474" }, { "path": "links.2.purpose", "from": null, "to": "text-mining" }, { "path": "links.2.return_type", "from": null, "to": "application/xml" }, { "path": "links.2.url", "from": null, "to": "https://www.jst.org.in/index.php/pub/article/download/532/3782" }, { "path": "links.3.purpose", "from": null, "to": "similarity-checking" }, { "path": "links.3.url", "from": null, "to": "https://www.jst.org.in/index.php/pub/article/view/532/474" }, { "path": "modes.0", "from": null, "to": "Visual" }, { "path": "record.issue_number", "from": null, "to": 1 }, { "path": "record.registered", "from": null, "to": "2026-09-23" }, { "path": "record.registrant", "from": null, "to": "Longman Publishers" }, { "path": "record.source", "from": null, "to": "crossref-api" }, { "path": "record.source_agency", "from": null, "to": "Crossref (member 25296)" }, { "path": "record.updated", "from": null, "to": "2026-09-27" }, { "path": "references.0.doi", "from": null, "to": "10.1109/wcre.1998.723173" }, { "path": "references.0.key", "from": null, "to": "ref1" }, { "path": "references.0.unstructured", "from": null, "to": "M. N. Armstrong and C. Trudeau, “Evaluating Architectural Extractors,” presented at Working Conference on Reverse Engineering, Honolulu, HI, pp. 30-39, October 12-14, 1998" }, { "path": "references.1.doi", "from": null, "to": "10.1002/0471028959.sof142" }, { "path": "references.1.key", "from": null, "to": "ref2" }, { "path": "references.1.unstructured", "from": null, "to": "Victor R. Basili, Gianluigi Caldiera, and H. Dieter Rombach, “The Goal Question Metric Approach,” in Encyclopedia of Software Engineering, Two Volume Set, Gianluigi Caldiera and Dieter H. Rombach, Eds. New York City: John Wiley and Sons, Inc., pp. 528- 532, 1994" }, { "path": "references.10.doi", "from": null, "to": "10.1145/290249.290255" }, { "path": "references.10.key", "from": null, "to": "ref11" }, { "path": "references.10.unstructured", "from": null, "to": "Barbara Ann Kitchenham, “Evaluating software engineering methods and tools. Parts 1 to 12.,” ACM SIGSOFT Software Engineering Notes, vol. 21-23, 1996-1998" }, { "path": "references.11.doi", "from": null, "to": "10.5897/ppr2013.0102" }, { "path": "references.11.key", "from": null, "to": "ref12" }, { "path": "references.11.unstructured", "from": null, "to": "Thomas S. Kuhn, The Structure of Scientific Revolutions, Third Edition. Chicago: The University of Chicago Press, 1996" }, { "path": "references.12.doi", "from": null, "to": "10.1109/icse.2001.919116" }, { "path": "references.12.key", "from": null, "to": "ref13" }, { "path": "references.12.unstructured", "from": null, "to": "William E. McUmber and Betty H. C. Cheng, “A Generic Framework for Formalizing UML,” presented at Proceeedings of the Twenty-third International Conference on Software Engineering, Toronto, Canada, pp. 433-442, 12-19 May 2001" }, { "path": "references.13.doi", "from": null, "to": "10.1145/279310.279314" }, { "path": "references.13.key", "from": null, "to": "ref14" }, { "path": "references.13.unstructured", "from": null, "to": "Gail C. Murphy, David Notkin, William G. Griswold, and Erica S. Lan, “An Empirical Study of Static Call Graph Extractors,” ACM Transactions on Software Engineering and Methodology, vol. 7, no. 2, pp. 158- 191, 1998" }, { "path": "references.14.doi", "from": null, "to": "10.1145/224155.565630" }, { "path": "references.14.key", "from": null, "to": "ref15" }, { "path": "references.14.unstructured", "from": null, "to": "Shari Lawrence Pfleeger, “Experimental Design and Analysis in Software Engineering, Parts 1-6,” ACM SIGSOFT Software Engineering Notes, vol. 19-20, 1994-1995" }, { "path": "references.15.doi", "from": null, "to": "10.1007/978-3-7091-6355-9_15" }, { "path": "references.15.key", "from": null, "to": "ref16" }, { "path": "references.15.unstructured", "from": null, "to": "M. Plath and M. Ryan, “SFI: A Feature Integration Tool,” in Advances in Computer Science, R. Berghammer and Y. Lakhnech, Eds. Heidelberg, Germany: Springer Verlag, pp. 201-216, 1999" }, { "path": "references.16.doi", "from": null, "to": "10.1109/wpc.2002.1021331" }, { "path": "references.16.key", "from": null, "to": "ref17" }, { "path": "references.16.unstructured", "from": null, "to": "Susan Elliott Sim, Richard C. Holt, and Steve Easterbrook, “On Using a Benchmark to Evaluate C++ Extractors,” presented at Tenth International Workshop on Program Comprehension, Paris, France, pp. 114-123" }, { "path": "references.17.doi", "from": null, "to": "10.1109/wcre.2000.891465" }, { "path": "references.17.key", "from": null, "to": "ref18" }, { "path": "references.17.unstructured", "from": null, "to": "Susan Elliott Sim and Margaret-Anne D. Storey, “A Structured Demonstration of Program Comprehension Tools,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 184-193, 23-25 November 2000" }, { "path": "references.18.doi", "from": null, "to": "10.1109/wcre.2000.891471" }, { "path": "references.18.key", "from": null, "to": "ref19" }, { "path": "references.18.unstructured", "from": null, "to": "Susan Elliott Sim, Margaret-Anne D. Storey, and Andreas Winter, “A Structured Demonstration of Five Program Comprehension Tools: Lessons Learnt,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 210-212, 23-25 November 2000" }, { "path": "references.19.doi", "from": null, "to": "10.1109/cmpass.1996.507877" }, { "path": "references.19.key", "from": null, "to": "ref20" }, { "path": "references.19.unstructured", "from": null, "to": "Tirumale Sreemani and Joanne M. Atlee, “Feasibility of Model Checking Software Requirements: A Case Study,” presented at Proceedings of COMPAS'96, Gaithersburg, Maryland, pp. 77-88, 17-21 June 1996" }, { "path": "references.2.doi", "from": null, "to": "10.1002/(sici)1096-908x(199809/10)10:5<305::aid-smr175>3.0.co;2-7" }, { "path": "references.2.key", "from": null, "to": "ref3" }, { "path": "references.2.unstructured", "from": null, "to": "Berndt Bellay and Harald Gall, “An Evaluation of Reverse Engineering Tool Capabilities,” Software Maintenance: Research and Practice, vol. 10, pp. 305-331, 1998" }, { "path": "references.20.doi", "from": null, "to": "10.1109/2.675631" }, { "path": "references.20.key", "from": null, "to": "ref21" }, { "path": "references.20.unstructured", "from": null, "to": "Walter F. Tichy, “Should Computer Scientists Experiment More?” IEEE Computer, pp. 32-40, May, 1998" }, { "path": "references.21.doi", "from": null, "to": "10.6028/nist.sp.500-246.overview" }, { "path": "references.21.key", "from": null, "to": "ref22" }, { "path": "references.21.unstructured", "from": null, "to": "Ellen M. Voorhees and Donna Harman, “Overview of the Eighth Text REtrieval Conference (TREC-8),” presented at Text REtrieval Conference (TREC-8), Gaithersberg, Maryland, pp. 1-24, November 17-19, 2000" }, { "path": "references.3.doi", "from": null, "to": "10.1016/s0950-5849(99)00082-8" }, { "path": "references.3.key", "from": null, "to": "ref4" }, { "path": "references.3.unstructured", "from": null, "to": "Ivan T. Bowman, Michael W. Godfrey, and Ric Holt, “Connecting Architecture Reconstruction Frameworks,” Journal of Information and Software Technology, vol. 42, no. 2, pp. 93-104, 1999" }, { "path": "references.4.doi", "from": null, "to": "10.1145/602497.602502" }, { "path": "references.4.key", "from": null, "to": "ref5" }, { "path": "references.4.unstructured", "from": null, "to": "Serge Demeyer, Tom Mens, and Michel Wermelinger, “Towards a Software Evolution Benchmark,” presented at Proceedings of the International Workshop on Principles of Software Evolution, Vienna, Austria, pp. 172-175, 10-11 September 2002" }, { "path": "references.5.doi", "from": null, "to": "10.1023/a:1008680612960" }, { "path": "references.5.key", "from": null, "to": "ref6" }, { "path": "references.5.unstructured", "from": null, "to": "Martin S. Feather, Stephen Fickas, Anthony Finkelstein, and Axel van Lamsweerde, “Requirements and Specification Exemplars,” Automated Software Engineering, vol. 4, pp. 419-438, 1997" }, { "path": "references.6.doi", "from": null, "to": "10.1109/wcre.2001.957809" }, { "path": "references.6.key", "from": null, "to": "ref7" }, { "path": "references.6.unstructured", "from": null, "to": "Rudolf Ferenc, Susan Elliott Sim, Richard C. Holt, Rainer Koschke, and Tibor Gyimóthy, “Towards a Standard Schema for C/C++,” presented at Eighth Working Conference on Reverse Engineering, Stuttgart, Germany, pp. 49-58, 2-5 October 2001" }, { "path": "references.7.doi", "from": null, "to": "10.1145/130951.1045493" }, { "path": "references.7.key", "from": null, "to": "ref8" }, { "path": "references.7.unstructured", "from": null, "to": "Jim Gray, “The Benchmark Handbook: For Database and Transaction Processing Systems,” San Mateo, CA: Morgan Kaufman Publishers, Inc, 1991" }, { "path": "references.8.doi", "from": null, "to": "10.1109/2.869367" }, { "path": "references.8.key", "from": null, "to": "ref9" }, { "path": "references.8.unstructured", "from": null, "to": "John L. Henning, “SPEC CPU2000: Measuring CPU Performance in the New Millennium,” IEEE Computer, pp. 28-35, July, 2000" }, { "path": "references.9.doi", "from": null, "to": "10.1109/wcre.2000.891463" }, { "path": "references.9.key", "from": null, "to": "ref10" }, { "path": "references.9.unstructured", "from": null, "to": "Richard C. Holt, Andreas Winter, and Andy Schürr, “GXL: Toward a Standard Exchange Format.,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 162-171, 23-25 November 2000" }, { "path": "referent", "from": null, "to": "Creation" }, { "path": "structural_type", "from": null, "to": "Digital" }, { "path": "titles.0.lang", "from": null, "to": "en" }, { "path": "titles.0.type", "from": null, "to": "PrincipalTitle" }, { "path": "titles.0.value", "from": null, "to": "Using Benchmarking to Advance Research: A Challenge to Software Engineering" }, { "path": "type", "from": null, "to": "JournalArticle" } ], "record_after": { "format": "smartscholars-doi-metadata/1.0", "doi": "10.46243/jst.2021.v6.i06.pp139-149", "referent": "Creation", "type": "JournalArticle", "structural_type": "Digital", "modes": [ "Visual" ], "characters": [ "Language" ], "titles": [ { "value": "Using Benchmarking to Advance Research: A Challenge to Software Engineering", "type": "PrincipalTitle", "lang": "en" } ], "identifiers": [ { "type": "DOI", "value": "10.46243/jst.2021.v6.i06.pp139-149" } ], "agents": [ { "role": "author", "name": { "given": "K Vijay", "family": "Kumar" }, "sequence": "first" }, { "role": "author", "name": { "given": "Dr.Y V R", "family": "Naga Pawan" }, "sequence": "additional" }, { "role": "publisher", "name": { "org": "Longman Publishers" } } ], "dates": { "published": "2021", "date_type": "PublicationDate", "online": "2021" }, "language": "en", "container": { "type": "Journal", "titles": [ { "value": "Journal of Science & Technology", "type": "PrincipalTitle" } ], "identifiers": [ { "type": "ISSN", "value": "2456-5660", "medium": "electronic" } ], "volume": "06", "issue": "06", "pages": { "first": "139", "last": "149" } }, "links": [ { "url": "https://www.jst.org.in/index.php/pub/article/view/532", "return_type": "text/html", "primary": true }, { "url": "https://www.jst.org.in/index.php/pub/article/download/532/474", "purpose": "text-mining", "return_type": "application/pdf" }, { "url": "https://www.jst.org.in/index.php/pub/article/download/532/3782", "purpose": "text-mining", "return_type": "application/xml" }, { "url": "https://www.jst.org.in/index.php/pub/article/view/532/474", "purpose": "similarity-checking" } ], "abstract": { "value": "Benchmarks have been used in computer science to compare the performance of computer systems, information retrieval algorithms, databases, and many other technologies. The creation and widespread use of a benchmark within a research area is frequently accompanied by rapid technical progress and community building. These observations have led us to formulate a theory of benchmarking within scientific disciplines. Based on this theory, we challenge software engineering research to become more scientific and cohesive by working as a community to define benchmarks. In support of this challenge, we present a case study of the reverse engineering community, where we have successfully used benchmarks to advance the state of research.", "lang": "en" }, "license": { "url": "https://creativecommons.org/licenses/by/4.0/", "start": "2021-01-01", "applies_to": "vor" }, "references": [ { "key": "ref1", "doi": "10.1109/wcre.1998.723173", "unstructured": "M. N. Armstrong and C. Trudeau, “Evaluating Architectural Extractors,” presented at Working Conference on Reverse Engineering, Honolulu, HI, pp. 30-39, October 12-14, 1998" }, { "key": "ref2", "doi": "10.1002/0471028959.sof142", "unstructured": "Victor R. Basili, Gianluigi Caldiera, and H. Dieter Rombach, “The Goal Question Metric Approach,” in Encyclopedia of Software Engineering, Two Volume Set, Gianluigi Caldiera and Dieter H. Rombach, Eds. New York City: John Wiley and Sons, Inc., pp. 528- 532, 1994" }, { "key": "ref3", "doi": "10.1002/(sici)1096-908x(199809/10)10:5<305::aid-smr175>3.0.co;2-7", "unstructured": "Berndt Bellay and Harald Gall, “An Evaluation of Reverse Engineering Tool Capabilities,” Software Maintenance: Research and Practice, vol. 10, pp. 305-331, 1998" }, { "key": "ref4", "doi": "10.1016/s0950-5849(99)00082-8", "unstructured": "Ivan T. Bowman, Michael W. Godfrey, and Ric Holt, “Connecting Architecture Reconstruction Frameworks,” Journal of Information and Software Technology, vol. 42, no. 2, pp. 93-104, 1999" }, { "key": "ref5", "doi": "10.1145/602497.602502", "unstructured": "Serge Demeyer, Tom Mens, and Michel Wermelinger, “Towards a Software Evolution Benchmark,” presented at Proceedings of the International Workshop on Principles of Software Evolution, Vienna, Austria, pp. 172-175, 10-11 September 2002" }, { "key": "ref6", "doi": "10.1023/a:1008680612960", "unstructured": "Martin S. Feather, Stephen Fickas, Anthony Finkelstein, and Axel van Lamsweerde, “Requirements and Specification Exemplars,” Automated Software Engineering, vol. 4, pp. 419-438, 1997" }, { "key": "ref7", "doi": "10.1109/wcre.2001.957809", "unstructured": "Rudolf Ferenc, Susan Elliott Sim, Richard C. Holt, Rainer Koschke, and Tibor Gyimóthy, “Towards a Standard Schema for C/C++,” presented at Eighth Working Conference on Reverse Engineering, Stuttgart, Germany, pp. 49-58, 2-5 October 2001" }, { "key": "ref8", "doi": "10.1145/130951.1045493", "unstructured": "Jim Gray, “The Benchmark Handbook: For Database and Transaction Processing Systems,” San Mateo, CA: Morgan Kaufman Publishers, Inc, 1991" }, { "key": "ref9", "doi": "10.1109/2.869367", "unstructured": "John L. Henning, “SPEC CPU2000: Measuring CPU Performance in the New Millennium,” IEEE Computer, pp. 28-35, July, 2000" }, { "key": "ref10", "doi": "10.1109/wcre.2000.891463", "unstructured": "Richard C. Holt, Andreas Winter, and Andy Schürr, “GXL: Toward a Standard Exchange Format.,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 162-171, 23-25 November 2000" }, { "key": "ref11", "doi": "10.1145/290249.290255", "unstructured": "Barbara Ann Kitchenham, “Evaluating software engineering methods and tools. Parts 1 to 12.,” ACM SIGSOFT Software Engineering Notes, vol. 21-23, 1996-1998" }, { "key": "ref12", "doi": "10.5897/ppr2013.0102", "unstructured": "Thomas S. Kuhn, The Structure of Scientific Revolutions, Third Edition. Chicago: The University of Chicago Press, 1996" }, { "key": "ref13", "doi": "10.1109/icse.2001.919116", "unstructured": "William E. McUmber and Betty H. C. Cheng, “A Generic Framework for Formalizing UML,” presented at Proceeedings of the Twenty-third International Conference on Software Engineering, Toronto, Canada, pp. 433-442, 12-19 May 2001" }, { "key": "ref14", "doi": "10.1145/279310.279314", "unstructured": "Gail C. Murphy, David Notkin, William G. Griswold, and Erica S. Lan, “An Empirical Study of Static Call Graph Extractors,” ACM Transactions on Software Engineering and Methodology, vol. 7, no. 2, pp. 158- 191, 1998" }, { "key": "ref15", "doi": "10.1145/224155.565630", "unstructured": "Shari Lawrence Pfleeger, “Experimental Design and Analysis in Software Engineering, Parts 1-6,” ACM SIGSOFT Software Engineering Notes, vol. 19-20, 1994-1995" }, { "key": "ref16", "doi": "10.1007/978-3-7091-6355-9_15", "unstructured": "M. Plath and M. Ryan, “SFI: A Feature Integration Tool,” in Advances in Computer Science, R. Berghammer and Y. Lakhnech, Eds. Heidelberg, Germany: Springer Verlag, pp. 201-216, 1999" }, { "key": "ref17", "doi": "10.1109/wpc.2002.1021331", "unstructured": "Susan Elliott Sim, Richard C. Holt, and Steve Easterbrook, “On Using a Benchmark to Evaluate C++ Extractors,” presented at Tenth International Workshop on Program Comprehension, Paris, France, pp. 114-123" }, { "key": "ref18", "doi": "10.1109/wcre.2000.891465", "unstructured": "Susan Elliott Sim and Margaret-Anne D. Storey, “A Structured Demonstration of Program Comprehension Tools,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 184-193, 23-25 November 2000" }, { "key": "ref19", "doi": "10.1109/wcre.2000.891471", "unstructured": "Susan Elliott Sim, Margaret-Anne D. Storey, and Andreas Winter, “A Structured Demonstration of Five Program Comprehension Tools: Lessons Learnt,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 210-212, 23-25 November 2000" }, { "key": "ref20", "doi": "10.1109/cmpass.1996.507877", "unstructured": "Tirumale Sreemani and Joanne M. Atlee, “Feasibility of Model Checking Software Requirements: A Case Study,” presented at Proceedings of COMPAS'96, Gaithersburg, Maryland, pp. 77-88, 17-21 June 1996" }, { "key": "ref21", "doi": "10.1109/2.675631", "unstructured": "Walter F. Tichy, “Should Computer Scientists Experiment More?” IEEE Computer, pp. 32-40, May, 1998" }, { "key": "ref22", "doi": "10.6028/nist.sp.500-246.overview", "unstructured": "Ellen M. Voorhees and Donna Harman, “Overview of the Eighth Text REtrieval Conference (TREC-8),” presented at Text REtrieval Conference (TREC-8), Gaithersberg, Maryland, pp. 1-24, November 17-19, 2000" } ], "record": { "registrant": "Longman Publishers", "registered": "2026-09-23", "updated": "2026-09-27", "issue_number": 1, "source": "crossref-api", "source_agency": "Crossref (member 25296)" } }, "url_after": "https://www.jst.org.in/index.php/pub/article/view/532", "sha256_before": null, "sha256_after": "8eea7b03488a669da1e5057b8604c85fc834c06d96bd2ac5f47ee75fc9038131" }, { "seq": 2, "kind": "update", "at": "2026-09-30 00:00:01", "by": "Administrator (admin)", "by_kind": "user", "note": "record re-read from api.crossref.org", "changes": [ { "path": "container.titles.0.value", "from": "Journal of Science & Technology", "to": "Journal of Science & Technology" } ], "record_after": { "format": "smartscholars-doi-metadata/1.0", "doi": "10.46243/jst.2021.v6.i06.pp139-149", "referent": "Creation", "type": "JournalArticle", "structural_type": "Digital", "modes": [ "Visual" ], "characters": [ "Language" ], "titles": [ { "value": "Using Benchmarking to Advance Research: A Challenge to Software Engineering", "type": "PrincipalTitle", "lang": "en" } ], "identifiers": [ { "type": "DOI", "value": "10.46243/jst.2021.v6.i06.pp139-149" } ], "agents": [ { "role": "author", "name": { "given": "K Vijay", "family": "Kumar" }, "sequence": "first" }, { "role": "author", "name": { "given": "Dr.Y V R", "family": "Naga Pawan" }, "sequence": "additional" }, { "role": "publisher", "name": { "org": "Longman Publishers" } } ], "dates": { "published": "2021", "date_type": "PublicationDate", "online": "2021" }, "language": "en", "container": { "type": "Journal", "titles": [ { "value": "Journal of Science & Technology", "type": "PrincipalTitle" } ], "identifiers": [ { "type": "ISSN", "value": "2456-5660", "medium": "electronic" } ], "volume": "06", "issue": "06", "pages": { "first": "139", "last": "149" } }, "links": [ { "url": "https://www.jst.org.in/index.php/pub/article/view/532", "return_type": "text/html", "primary": true }, { "url": "https://www.jst.org.in/index.php/pub/article/download/532/474", "purpose": "text-mining", "return_type": "application/pdf" }, { "url": "https://www.jst.org.in/index.php/pub/article/download/532/3782", "purpose": "text-mining", "return_type": "application/xml" }, { "url": "https://www.jst.org.in/index.php/pub/article/view/532/474", "purpose": "similarity-checking" } ], "abstract": { "value": "Benchmarks have been used in computer science to compare the performance of computer systems, information retrieval algorithms, databases, and many other technologies. The creation and widespread use of a benchmark within a research area is frequently accompanied by rapid technical progress and community building. These observations have led us to formulate a theory of benchmarking within scientific disciplines. Based on this theory, we challenge software engineering research to become more scientific and cohesive by working as a community to define benchmarks. In support of this challenge, we present a case study of the reverse engineering community, where we have successfully used benchmarks to advance the state of research.", "lang": "en" }, "license": { "url": "https://creativecommons.org/licenses/by/4.0/", "start": "2021-01-01", "applies_to": "vor" }, "references": [ { "key": "ref1", "doi": "10.1109/wcre.1998.723173", "unstructured": "M. N. Armstrong and C. Trudeau, “Evaluating Architectural Extractors,” presented at Working Conference on Reverse Engineering, Honolulu, HI, pp. 30-39, October 12-14, 1998" }, { "key": "ref2", "doi": "10.1002/0471028959.sof142", "unstructured": "Victor R. Basili, Gianluigi Caldiera, and H. Dieter Rombach, “The Goal Question Metric Approach,” in Encyclopedia of Software Engineering, Two Volume Set, Gianluigi Caldiera and Dieter H. Rombach, Eds. New York City: John Wiley and Sons, Inc., pp. 528- 532, 1994" }, { "key": "ref3", "doi": "10.1002/(sici)1096-908x(199809/10)10:5<305::aid-smr175>3.0.co;2-7", "unstructured": "Berndt Bellay and Harald Gall, “An Evaluation of Reverse Engineering Tool Capabilities,” Software Maintenance: Research and Practice, vol. 10, pp. 305-331, 1998" }, { "key": "ref4", "doi": "10.1016/s0950-5849(99)00082-8", "unstructured": "Ivan T. Bowman, Michael W. Godfrey, and Ric Holt, “Connecting Architecture Reconstruction Frameworks,” Journal of Information and Software Technology, vol. 42, no. 2, pp. 93-104, 1999" }, { "key": "ref5", "doi": "10.1145/602497.602502", "unstructured": "Serge Demeyer, Tom Mens, and Michel Wermelinger, “Towards a Software Evolution Benchmark,” presented at Proceedings of the International Workshop on Principles of Software Evolution, Vienna, Austria, pp. 172-175, 10-11 September 2002" }, { "key": "ref6", "doi": "10.1023/a:1008680612960", "unstructured": "Martin S. Feather, Stephen Fickas, Anthony Finkelstein, and Axel van Lamsweerde, “Requirements and Specification Exemplars,” Automated Software Engineering, vol. 4, pp. 419-438, 1997" }, { "key": "ref7", "doi": "10.1109/wcre.2001.957809", "unstructured": "Rudolf Ferenc, Susan Elliott Sim, Richard C. Holt, Rainer Koschke, and Tibor Gyimóthy, “Towards a Standard Schema for C/C++,” presented at Eighth Working Conference on Reverse Engineering, Stuttgart, Germany, pp. 49-58, 2-5 October 2001" }, { "key": "ref8", "doi": "10.1145/130951.1045493", "unstructured": "Jim Gray, “The Benchmark Handbook: For Database and Transaction Processing Systems,” San Mateo, CA: Morgan Kaufman Publishers, Inc, 1991" }, { "key": "ref9", "doi": "10.1109/2.869367", "unstructured": "John L. Henning, “SPEC CPU2000: Measuring CPU Performance in the New Millennium,” IEEE Computer, pp. 28-35, July, 2000" }, { "key": "ref10", "doi": "10.1109/wcre.2000.891463", "unstructured": "Richard C. Holt, Andreas Winter, and Andy Schürr, “GXL: Toward a Standard Exchange Format.,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 162-171, 23-25 November 2000" }, { "key": "ref11", "doi": "10.1145/290249.290255", "unstructured": "Barbara Ann Kitchenham, “Evaluating software engineering methods and tools. Parts 1 to 12.,” ACM SIGSOFT Software Engineering Notes, vol. 21-23, 1996-1998" }, { "key": "ref12", "doi": "10.5897/ppr2013.0102", "unstructured": "Thomas S. Kuhn, The Structure of Scientific Revolutions, Third Edition. Chicago: The University of Chicago Press, 1996" }, { "key": "ref13", "doi": "10.1109/icse.2001.919116", "unstructured": "William E. McUmber and Betty H. C. Cheng, “A Generic Framework for Formalizing UML,” presented at Proceeedings of the Twenty-third International Conference on Software Engineering, Toronto, Canada, pp. 433-442, 12-19 May 2001" }, { "key": "ref14", "doi": "10.1145/279310.279314", "unstructured": "Gail C. Murphy, David Notkin, William G. Griswold, and Erica S. Lan, “An Empirical Study of Static Call Graph Extractors,” ACM Transactions on Software Engineering and Methodology, vol. 7, no. 2, pp. 158- 191, 1998" }, { "key": "ref15", "doi": "10.1145/224155.565630", "unstructured": "Shari Lawrence Pfleeger, “Experimental Design and Analysis in Software Engineering, Parts 1-6,” ACM SIGSOFT Software Engineering Notes, vol. 19-20, 1994-1995" }, { "key": "ref16", "doi": "10.1007/978-3-7091-6355-9_15", "unstructured": "M. Plath and M. Ryan, “SFI: A Feature Integration Tool,” in Advances in Computer Science, R. Berghammer and Y. Lakhnech, Eds. Heidelberg, Germany: Springer Verlag, pp. 201-216, 1999" }, { "key": "ref17", "doi": "10.1109/wpc.2002.1021331", "unstructured": "Susan Elliott Sim, Richard C. Holt, and Steve Easterbrook, “On Using a Benchmark to Evaluate C++ Extractors,” presented at Tenth International Workshop on Program Comprehension, Paris, France, pp. 114-123" }, { "key": "ref18", "doi": "10.1109/wcre.2000.891465", "unstructured": "Susan Elliott Sim and Margaret-Anne D. Storey, “A Structured Demonstration of Program Comprehension Tools,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 184-193, 23-25 November 2000" }, { "key": "ref19", "doi": "10.1109/wcre.2000.891471", "unstructured": "Susan Elliott Sim, Margaret-Anne D. Storey, and Andreas Winter, “A Structured Demonstration of Five Program Comprehension Tools: Lessons Learnt,” presented at Seventh Working Conference on Reverse Engineering, Brisbane, Queensland, Australia, pp. 210-212, 23-25 November 2000" }, { "key": "ref20", "doi": "10.1109/cmpass.1996.507877", "unstructured": "Tirumale Sreemani and Joanne M. Atlee, “Feasibility of Model Checking Software Requirements: A Case Study,” presented at Proceedings of COMPAS'96, Gaithersburg, Maryland, pp. 77-88, 17-21 June 1996" }, { "key": "ref21", "doi": "10.1109/2.675631", "unstructured": "Walter F. Tichy, “Should Computer Scientists Experiment More?” IEEE Computer, pp. 32-40, May, 1998" }, { "key": "ref22", "doi": "10.6028/nist.sp.500-246.overview", "unstructured": "Ellen M. Voorhees and Donna Harman, “Overview of the Eighth Text REtrieval Conference (TREC-8),” presented at Text REtrieval Conference (TREC-8), Gaithersberg, Maryland, pp. 1-24, November 17-19, 2000" } ], "record": { "registrant": "Longman Publishers", "registered": "2026-09-23", "updated": "2026-09-27", "issue_number": 1, "source": "crossref-api", "source_agency": "Crossref (member 25296)" } }, "url_after": "https://www.jst.org.in/index.php/pub/article/view/532", "sha256_before": "8eea7b03488a669da1e5057b8604c85fc834c06d96bd2ac5f47ee75fc9038131", "sha256_after": "2cf2633011cd7cc07e40eed358243262807c18dd95ad704c7464ad4fa3657678" } ] }