{"observation":{"id":"7ec0595e-ec84-427a-a9c7-22c290258f01","tool":"extracta-ai","tool_name":"Extracta.ai","criterion":"noise-in-output","criterion_name":"Noise in output","criterion_definition":"Does it add incorrect or hallucinated fields?","criterion_evidence_type":"transformation","criterion_rank_role":"decisive","criterion_rank_role_reason":"Incorrect or hallucinated fields mean the resume parser is not producing reliable structured data, which is central to the job. (3 of 3 judges)","scenario":"messy-real-world-resume-john-kumar","scenario_name":"Messy real-world resume — John Kumar","group_tag":"resume-parsing","scenario_description":"A poorly formatted, inconsistent real-world resume for John Kumar, used to test robustness against noisy structure, inconsistent dates, and mixed-content sections.","modality":"pdf","input_text":null,"input_artifact_refs":[{"alt":null,"url":"https://d3epheqghktydj.cloudfront.net/parseur-input3-john-kumar-messyresume-75275a848042.pdf","role":"input"}],"stresses":["messy formatting robustness","section detection fallback","inconsistent date parsing","soft-skills extraction","noise and hallucination control"],"verdict":"struggled","score":null,"score_total":null,"note":"Emits a blank Languages item when the resume has no languages section, leaving an empty placeholder instead of omitting the field.","evidence_state":"verified","source":null,"artifacts":[{"url":"https://d3epheqghktydj.cloudfront.net/research-media-screenshot-202026-05-05-20124544-174ecd83f60c.png","role":"input","alt":"Research media screenshot 202026 05 05 20124544.png"},{"url":"https://d3epheqghktydj.cloudfront.net/research-media-image-65b5b893321e.png","role":"output","alt":null}],"run_id":"cbbef4db-964c-49fa-a57f-a2977822bdfc","study_title":"Parse resumes into structured data using an API","study_kind":"generation","research_task":"86b9jm30n","tested_at":null,"completeness":"input-and-output","input":{"state":"files","text":null,"files":[{"url":"https://d3epheqghktydj.cloudfront.net/parseur-input3-john-kumar-messyresume-75275a848042.pdf","filename":"parseur-input3-john-kumar-messyresume-75275a848042.pdf","alt":"Messy real-world resume — John Kumar","role":"input"}],"modality":"pdf","stresses":["messy formatting robustness","section detection fallback","inconsistent date parsing","soft-skills extraction","noise and hallucination control"]},"tool_page_slug":"extracta-labs","tool_url":"https://aidemos.com/tools/extracta-labs","permalink":"https://aidemos.com/evidence/7ec0595e-ec84-427a-a9c7-22c290258f01","api_url":"https://ai.aidemos.com/v1/observations/7ec0595e-ec84-427a-a9c7-22c290258f01"},"peers":[{"id":"490221bf-4f71-43fd-a88a-fe2777655a90","tool":"affinda","tool_name":"Affinda","verdict":"failed","score":null,"score_total":null,"note":"The skills output includes hallucinated taxonomy noise, such as Business Education, that does not appear in the resume.","artifact_count":2,"thumbnail":"https://d3epheqghktydj.cloudfront.net/research-media-image-d6a62fc24445.png","evidence_url":"https://aidemos.com/evidence/490221bf-4f71-43fd-a88a-fe2777655a90"},{"id":"f4a9dbe8-9c63-425b-a0b9-39c490e6965a","tool":"cvparserpro","tool_name":"CVParserPro","verdict":"failed","score":null,"score_total":null,"note":"The parser hallucinated unsupported month values in the work date range, expanding 2019–2021 into January 2019 to December 2021.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/cvparserpro-p7-month-hallucination-e794ae0ec6e7.png","evidence_url":"https://aidemos.com/evidence/f4a9dbe8-9c63-425b-a0b9-39c490e6965a"},{"id":"53baa4b9-f590-4704-af4d-1715255eb658","tool":"hireability","tool_name":"Hireability","verdict":"failed","score":null,"score_total":null,"note":"The certification field merged three content types — certifications, skills, and references — into one extracted value instead of separating them.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/hireability-hireability-task3-certs-refs-merged-mock-c80cee78a881.png","evidence_url":"https://aidemos.com/evidence/53baa4b9-f590-4704-af4d-1715255eb658"},{"id":"88d0b110-6c68-48a4-be94-bb032e62ab16","tool":"hrflow","tool_name":"HrFlow","verdict":"mixed","score":null,"score_total":null,"note":"The messy parse can bleed location text into a task item, prepending 'Pune' to one extracted responsibility.","artifact_count":0,"thumbnail":null,"evidence_url":"https://aidemos.com/evidence/88d0b110-6c68-48a4-be94-bb032e62ab16"},{"id":"f3504830-9157-4146-a34d-a211439d3ce7","tool":"skima-ai","tool_name":"Skima AI","verdict":"failed","score":null,"score_total":null,"note":"The messy-resume skills block was collapsed into a single concatenated string with no separators, making the field hard to reuse downstream.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/skima-ai-09-input3-skills-concatenated-mock-d25303702406.png","evidence_url":"https://aidemos.com/evidence/f3504830-9157-4146-a34d-a211439d3ce7"}],"other_criteria":[{"id":"ac039802-7f46-4988-9c6f-e70dd10c5931","criterion":"accuracy","criterion_name":"Accuracy","rank_role":"decisive","verdict":"mixed","score":null,"score_total":null,"note":"Preserves the education marks text verbatim, including the unnormalised '72 percent marks' value instead of converting it to a cleaner percentage format.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/ac039802-7f46-4988-9c6f-e70dd10c5931"},{"id":"95b1a04c-460c-4ce9-92a3-12b5c95a28df","criterion":"accuracy","criterion_name":"Accuracy","rank_role":"decisive","verdict":"mixed","score":null,"score_total":null,"note":"Returns both certification names in lowercase exactly as written, with no capitalization normalisation applied.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/95b1a04c-460c-4ce9-92a3-12b5c95a28df"},{"id":"d85153f5-b16a-4009-9e33-65440ad1a387","criterion":"field-coverage","criterion_name":"Field coverage","rank_role":"decisive","verdict":"worked","score":null,"score_total":null,"note":"Still covers the main resume fields on the messy input: name, email, phone, experience, education, skills, and certifications.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/d85153f5-b16a-4009-9e33-65440ad1a387"},{"id":"87069ea1-3f6c-451a-93a0-147d3a3cc323","criterion":"messy-resume-handling","criterion_name":"Messy resume handling","rank_role":"decisive","verdict":"worked","score":null,"score_total":null,"note":"Handles the noisy, inconsistent layout without crashing and still extracts the main sections.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/87069ea1-3f6c-451a-93a0-147d3a3cc323"},{"id":"5ecc3f0c-d625-475d-8251-0b44a2f88196","criterion":"output-quality","criterion_name":"Output quality","rank_role":"decisive","verdict":"worked","score":null,"score_total":null,"note":"Produces a usable JSON result even on the messy resume, with the important sections still readable and structured.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/5ecc3f0c-d625-475d-8251-0b44a2f88196"}],"appears_in":[{"page_type":"ranking","slug":"resume-parsing-api","title":"Best AI Tools for Parsing Resumes via API (2026)","url":"https://aidemos.com/best/resume-parsing-api","binding":"run"}],"same_scenario":[{"id":"490221bf-4f71-43fd-a88a-fe2777655a90","tool":"affinda","tool_name":"Affinda","verdict":"failed","score":null,"score_total":null,"note":"The skills output includes hallucinated taxonomy noise, such as Business Education, that does not appear in the resume."},{"id":"f4a9dbe8-9c63-425b-a0b9-39c490e6965a","tool":"cvparserpro","tool_name":"CVParserPro","verdict":"failed","score":null,"score_total":null,"note":"The parser hallucinated unsupported month values in the work date range, expanding 2019–2021 into January 2019 to December 2021."},{"id":"53baa4b9-f590-4704-af4d-1715255eb658","tool":"hireability","tool_name":"Hireability","verdict":"failed","score":null,"score_total":null,"note":"The certification field merged three content types — certifications, skills, and references — into one extracted value instead of separating them."},{"id":"88d0b110-6c68-48a4-be94-bb032e62ab16","tool":"hrflow","tool_name":"HrFlow","verdict":"mixed","score":null,"score_total":null,"note":"The messy parse can bleed location text into a task item, prepending 'Pune' to one extracted responsibility."},{"id":"f3504830-9157-4146-a34d-a211439d3ce7","tool":"skima-ai","tool_name":"Skima AI","verdict":"failed","score":null,"score_total":null,"note":"The messy-resume skills block was collapsed into a single concatenated string with no separators, making the field hard to reuse downstream."}]}