{"observation":{"id":"a172fc19-f483-4799-87f6-2a103cfbe529","tool":"grok","tool_name":"Grok","criterion":"code-generation-quality","criterion_name":"Code Generation Quality","criterion_definition":"Syntactically correct, runnable on first attempt.","criterion_evidence_type":"transformation","criterion_rank_role":"decisive","criterion_rank_role_reason":"If the generated code is not syntactically correct and runnable, the tool has failed at producing the animation at all. (3 of 3 judges)","scenario":"search-engines-basics-explainer","scenario_name":"Search engines basics explainer","group_tag":null,"scenario_description":"A vague, linear educational animation prompt asking the tool to explain how search engines work with minimal guidance. Designed to stress inference, scene structuring, pacing, and coherent visual flow from an underspecified request.","modality":"text","input_text":"Create an animation video explaining how search engines work.","input_artifact_refs":[],"stresses":[],"verdict":"worked","score":null,"score_total":null,"note":"Produced syntactically correct HTML/CSS/JavaScript code on the first pass.","evidence_state":"observed","source":"first-party","artifacts":[{"url":"https://d3epheqghktydj.cloudfront.net/grok-screen-recording-2026-05-04-135441.mp4","role":"context","alt":"Grok — Screen Recording 2026-05-04 135441.mp4"}],"run_id":"c751a76b-a105-4b3e-9ff3-6c4d6adc9f04","study_title":"Generate Code-based Animations from text inputs","study_kind":"backfill","research_task":"86b9nnzmy","tested_at":"2026-06-24T08:34:34.142000+00:00","completeness":"input-and-output","input":{"state":"text","text":"Create an animation video explaining how search engines work.","files":[],"modality":"text","stresses":[]},"tool_page_slug":null,"tool_url":null,"permalink":"https://aidemos.com/evidence/a172fc19-f483-4799-87f6-2a103cfbe529","api_url":"https://ai.aidemos.com/v1/observations/a172fc19-f483-4799-87f6-2a103cfbe529"},"peers":[{"id":"7062524a-d54a-4a4c-98b2-dd205aff10bb","tool":"animg","tool_name":"AnimG","verdict":"worked","score":null,"score_total":null,"note":"Generates syntactically correct Manim code on the first pass.","artifact_count":0,"thumbnail":null,"evidence_url":"https://aidemos.com/evidence/7062524a-d54a-4a4c-98b2-dd205aff10bb"},{"id":"783827be-868d-4058-9a88-b7fda3c34533","tool":"chatgpt-canvas","tool_name":"ChatGPT Canvas","verdict":"worked","score":null,"score_total":null,"note":"Generated syntactically correct, runnable HTML/CSS/JavaScript on the first attempt for the search-engine explainer.","artifact_count":2,"thumbnail":"https://d3epheqghktydj.cloudfront.net/chatgpt-canvas-screen-recording-2026-04-30-170046.mp4","evidence_url":"https://aidemos.com/evidence/783827be-868d-4058-9a88-b7fda3c34533"},{"id":"66a96576-7f89-45d6-9960-686650ac8834","tool":"claude-artifacts","tool_name":"Claude Artifacts","verdict":"worked","score":null,"score_total":null,"note":"It generated a runnable webpage on the first attempt, with autoplay working on load.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/claude-artifacts-screen-recording-2026-04-30-141425.mp4","evidence_url":"https://aidemos.com/evidence/66a96576-7f89-45d6-9960-686650ac8834"},{"id":"23138cac-317e-4e2a-a9a8-f9b791df45e4","tool":"gemini-canvas","tool_name":"Gemini Canvas","verdict":"worked","score":null,"score_total":null,"note":"Produces syntactically correct HTML/CSS/JavaScript on the first attempt for a basic explainer animation.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/gemini-canvas-screen-recording-2026-05-04-120930.mp4","evidence_url":"https://aidemos.com/evidence/23138cac-317e-4e2a-a9a8-f9b791df45e4"},{"id":"22986790-e8b3-4c13-a6e7-7e750f1d8709","tool":"replit","tool_name":"Replit","verdict":"worked","score":null,"score_total":null,"note":"Produces runnable React animation code on the first attempt.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/replit-search-engine-explainer-apr-18-15-53-35.mp4","evidence_url":"https://aidemos.com/evidence/22986790-e8b3-4c13-a6e7-7e750f1d8709"}],"other_criteria":[{"id":"2312144f-75e5-45c5-91b0-3f14ad09a663","criterion":"input-handling","criterion_name":"Input Handling","rank_role":"decisive","verdict":"worked","score":null,"score_total":null,"note":"Accepted a fully specified HTML/CSS/JavaScript animation prompt immediately, but still misread part of the request as interactive webpage behavior rather than animation-first motion graphics.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/2312144f-75e5-45c5-91b0-3f14ad09a663"},{"id":"0e15c1ce-6c53-4c57-b915-56b788c78777","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"struggled","score":null,"score_total":null,"note":"The output read more like an interactive webpage than an animation video: it included interactive elements despite the no-interaction requirement, had cramped spacing and overlapping layout issues, and lacked clear visual hierarchy.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/0e15c1ce-6c53-4c57-b915-56b788c78777"},{"id":"e8505de0-5cac-48fb-811d-29a4753d36b6","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"failed","score":null,"score_total":null,"note":"Produces output that reads more like an interactive webpage than a video-like animation, with cramped spacing, overlapping elements, weak visual hierarchy, and visible controls.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/e8505de0-5cac-48fb-811d-29a4753d36b6"},{"id":"196fc12d-b1d5-46bc-823b-797983bce1b4","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"failed","score":null,"score_total":null,"note":"Produces a cluttered, webpage-like result with overlapping elements, cramped spacing, and no clear motion-graphics hierarchy.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/196fc12d-b1d5-46bc-823b-797983bce1b4"}],"appears_in":[{"page_type":"ranking","slug":"code-animation-generators","title":"Best AI Tools for Generating Code-Based Animations from Text Prompts","url":"https://aidemos.com/best/code-animation-generators","binding":"run"},{"page_type":"use-case","slug":"generate-code-animation-videos","title":"Generate Code-Based Animation Videos from Text Prompts Using AI","url":"https://aidemos.com/use-cases/generate-code-animation-videos","binding":"run"}],"same_scenario":[{"id":"7062524a-d54a-4a4c-98b2-dd205aff10bb","tool":"animg","tool_name":"AnimG","verdict":"worked","score":null,"score_total":null,"note":"Generates syntactically correct Manim code on the first pass."},{"id":"783827be-868d-4058-9a88-b7fda3c34533","tool":"chatgpt-canvas","tool_name":"ChatGPT Canvas","verdict":"worked","score":null,"score_total":null,"note":"Generated syntactically correct, runnable HTML/CSS/JavaScript on the first attempt for the search-engine explainer."},{"id":"66a96576-7f89-45d6-9960-686650ac8834","tool":"claude-artifacts","tool_name":"Claude Artifacts","verdict":"worked","score":null,"score_total":null,"note":"It generated a runnable webpage on the first attempt, with autoplay working on load."},{"id":"23138cac-317e-4e2a-a9a8-f9b791df45e4","tool":"gemini-canvas","tool_name":"Gemini Canvas","verdict":"worked","score":null,"score_total":null,"note":"Produces syntactically correct HTML/CSS/JavaScript on the first attempt for a basic explainer animation."},{"id":"22986790-e8b3-4c13-a6e7-7e750f1d8709","tool":"replit","tool_name":"Replit","verdict":"worked","score":null,"score_total":null,"note":"Produces runnable React animation code on the first attempt."}]}