{"observation":{"id":"8c586228-b763-4fd1-9403-2f83e2b9bcfe","tool":"chatgpt-canvas","tool_name":"ChatGPT Canvas","criterion":"code-generation-quality","criterion_name":"Code Generation Quality","criterion_definition":"Syntactically correct, runnable on first attempt.","criterion_evidence_type":"transformation","criterion_rank_role":"decisive","criterion_rank_role_reason":"If the generated code is not syntactically correct and runnable, the tool has failed at producing the animation at all. (3 of 3 judges)","scenario":"search-engines-basics-explainer","scenario_name":"Search engines basics explainer","group_tag":null,"scenario_description":"A vague, linear educational animation prompt asking the tool to explain how search engines work with minimal guidance. Designed to stress inference, scene structuring, pacing, and coherent visual flow from an underspecified request.","modality":"text","input_text":"Create an animation video explaining how search engines work.","input_artifact_refs":[],"stresses":[],"verdict":"worked","score":null,"score_total":null,"note":"Generates syntactically correct HTML/CSS/JavaScript on the first attempt and can produce an auto-playing, continuous-loop animation without user interaction.","evidence_state":"verified","source":"first-party","artifacts":[{"url":"https://d3epheqghktydj.cloudfront.net/chatgpt-canvas-chatgpt-search-engine-animation.mp4","role":"output","alt":"ChatGPT Canvas — chatgpt-search-engine-animation.mp4"}],"run_id":"c751a76b-a105-4b3e-9ff3-6c4d6adc9f04","study_title":"Generate Code-based Animations from text inputs","study_kind":"backfill","research_task":"86b9nnzmy","tested_at":"2026-06-24T08:34:34.142000+00:00","completeness":"input-and-output","input":{"state":"text","text":"Create an animation video explaining how search engines work.","files":[],"modality":"text","stresses":[]},"tool_page_slug":null,"tool_url":null,"permalink":"https://aidemos.com/evidence/8c586228-b763-4fd1-9403-2f83e2b9bcfe","api_url":"https://ai.aidemos.com/v1/observations/8c586228-b763-4fd1-9403-2f83e2b9bcfe"},"peers":[{"id":"7062524a-d54a-4a4c-98b2-dd205aff10bb","tool":"animg","tool_name":"AnimG","verdict":"worked","score":null,"score_total":null,"note":"Generates syntactically correct Manim code on the first pass.","artifact_count":0,"thumbnail":null,"evidence_url":"https://aidemos.com/evidence/7062524a-d54a-4a4c-98b2-dd205aff10bb"},{"id":"66a96576-7f89-45d6-9960-686650ac8834","tool":"claude-artifacts","tool_name":"Claude Artifacts","verdict":"worked","score":null,"score_total":null,"note":"It generated a runnable webpage on the first attempt, with autoplay working on load.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/claude-artifacts-screen-recording-2026-04-30-141425.mp4","evidence_url":"https://aidemos.com/evidence/66a96576-7f89-45d6-9960-686650ac8834"},{"id":"23138cac-317e-4e2a-a9a8-f9b791df45e4","tool":"gemini-canvas","tool_name":"Gemini Canvas","verdict":"worked","score":null,"score_total":null,"note":"Produces syntactically correct HTML/CSS/JavaScript on the first attempt for a basic explainer animation.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/gemini-canvas-screen-recording-2026-05-04-120930.mp4","evidence_url":"https://aidemos.com/evidence/23138cac-317e-4e2a-a9a8-f9b791df45e4"},{"id":"a172fc19-f483-4799-87f6-2a103cfbe529","tool":"grok","tool_name":"Grok","verdict":"worked","score":null,"score_total":null,"note":"Produced syntactically correct HTML/CSS/JavaScript code on the first pass.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/grok-screen-recording-2026-05-04-135441.mp4","evidence_url":"https://aidemos.com/evidence/a172fc19-f483-4799-87f6-2a103cfbe529"},{"id":"22986790-e8b3-4c13-a6e7-7e750f1d8709","tool":"replit","tool_name":"Replit","verdict":"worked","score":null,"score_total":null,"note":"Produces runnable React animation code on the first attempt.","artifact_count":1,"thumbnail":"https://d3epheqghktydj.cloudfront.net/replit-search-engine-explainer-apr-18-15-53-35.mp4","evidence_url":"https://aidemos.com/evidence/22986790-e8b3-4c13-a6e7-7e750f1d8709"}],"other_criteria":[{"id":"d60b7cdd-b974-41b0-bec9-1c6a3eee97b0","criterion":"in-platform-preview","criterion_name":"In-Platform Preview","rank_role":"context","verdict":"worked","score":null,"score_total":null,"note":"Canvas showed the animation immediately on load, with live preview available and the motion running in a continuous loop.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/d60b7cdd-b974-41b0-bec9-1c6a3eee97b0"},{"id":"cdfe3d65-164a-4c24-a556-0d75b593be5c","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"mixed","score":null,"score_total":null,"note":"Can match the requested topic, but the first-pass visual output is only a plain horizontal flowchart with text-only nodes, overflowing labels, and no icons or hierarchy, so it is not visually polished.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/cdfe3d65-164a-4c24-a556-0d75b593be5c"},{"id":"df5fad76-48f1-4a26-8164-a82ffcbc7976","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"struggled","score":null,"score_total":null,"note":"The first-pass visual output was only a plain horizontal flowchart: text overflowed node boxes, there were no icons or visual hierarchy, and component differentiation was minimal.","artifact_count":2,"evidence_url":"https://aidemos.com/evidence/df5fad76-48f1-4a26-8164-a82ffcbc7976"},{"id":"dc02827e-6c3e-4881-bbfe-8eb3214090b3","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"struggled","score":null,"score_total":null,"note":"The initial output was a plain horizontal flowchart with text-only nodes; labels overflowed the node containers and there were no icons or visual hierarchy.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/dc02827e-6c3e-4881-bbfe-8eb3214090b3"},{"id":"fe204dc8-c009-42bf-9962-1b563998736e","criterion":"output-quality","criterion_name":"Output Quality","rank_role":"decisive","verdict":"struggled","score":null,"score_total":null,"note":"Can explain the search-engine pipeline accurately, but first-pass output is visually weak: a plain horizontal flowchart with overflowing node text, no icons, no visual hierarchy, and minimal differentiation between components.","artifact_count":1,"evidence_url":"https://aidemos.com/evidence/fe204dc8-c009-42bf-9962-1b563998736e"}],"appears_in":[{"page_type":"ranking","slug":"code-animation-generators","title":"Best AI Tools for Generating Code-Based Animations from Text Prompts","url":"https://aidemos.com/best/code-animation-generators","binding":"run"},{"page_type":"use-case","slug":"generate-code-animation-videos","title":"Generate Code-Based Animation Videos from Text Prompts Using AI","url":"https://aidemos.com/use-cases/generate-code-animation-videos","binding":"run"}],"same_scenario":[{"id":"7062524a-d54a-4a4c-98b2-dd205aff10bb","tool":"animg","tool_name":"AnimG","verdict":"worked","score":null,"score_total":null,"note":"Generates syntactically correct Manim code on the first pass."},{"id":"66a96576-7f89-45d6-9960-686650ac8834","tool":"claude-artifacts","tool_name":"Claude Artifacts","verdict":"worked","score":null,"score_total":null,"note":"It generated a runnable webpage on the first attempt, with autoplay working on load."},{"id":"23138cac-317e-4e2a-a9a8-f9b791df45e4","tool":"gemini-canvas","tool_name":"Gemini Canvas","verdict":"worked","score":null,"score_total":null,"note":"Produces syntactically correct HTML/CSS/JavaScript on the first attempt for a basic explainer animation."},{"id":"a172fc19-f483-4799-87f6-2a103cfbe529","tool":"grok","tool_name":"Grok","verdict":"worked","score":null,"score_total":null,"note":"Produced syntactically correct HTML/CSS/JavaScript code on the first pass."},{"id":"22986790-e8b3-4c13-a6e7-7e750f1d8709","tool":"replit","tool_name":"Replit","verdict":"worked","score":null,"score_total":null,"note":"Produces runnable React animation code on the first attempt."}]}