-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathindex.html
More file actions
7 lines (7 loc) · 175 KB
/
Copy pathindex.html
File metadata and controls
7 lines (7 loc) · 175 KB
1
2
3
4
5
6
7
<!DOCTYPE html><html lang="en" class="silkscreen_5c214b00-module__BDnqEG__variable plexmono_301f1454-module__3YJGwG__variable"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="./_next/static/media/PlexMono_Regular-s.p.2x6m9-apme3b_.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="./_next/static/media/Silkscreen_Regular-s.p.3es8zrs5x_4vu.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="./branding/computational-robotics.jpg" as="image"/><link rel="preload" as="image" href="./pipeline/garden/preview.webp"/><link rel="preload" as="image" href="./pipeline/lab/preview.webp"/><link rel="preload" as="image" href="./pipeline/kitchen2/preview.webp"/><link rel="preload" as="image" href="./pipeline/worldlabs/preview.webp"/><link rel="preload" as="image" href="./pipeline/observatory/preview.webp"/><link rel="stylesheet" href="./_next/static/chunks/1zf8qs8h9vuqk.css" data-precedence="next"/><link rel="preload" as="script" fetchPriority="low" href="./_next/static/chunks/0ba1ebj8wcm3s.js"/><script src="./_next/static/chunks/2turxypa9qaj4.js" async=""></script><script src="./_next/static/chunks/01yyx2dlde7l1.js" async=""></script><script src="./_next/static/chunks/turbopack-063p7s1qohf7o.js" async=""></script><script src="./_next/static/chunks/3d-g45tvvtwds.js" async=""></script><script src="./_next/static/chunks/24ble59yb_9_o.js" async=""></script><script src="./_next/static/chunks/36hy0ec4-ck76.js" async=""></script><script src="./_next/static/chunks/0p_6srangbw9h.js" async=""></script><script src="./_next/static/chunks/0_sx6di6mxk5o.js" async=""></script><meta name="next-size-adjust" content=""/><title>SceneAgent | Harvard Computational Robotics Group</title><meta name="description" content="SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training."/><link rel="author" href="https://computationalrobotics.seas.harvard.edu/"/><meta name="author" content="Luke Hollis"/><link rel="author" href="https://computationalrobotics.seas.harvard.edu/"/><meta name="author" content="Tianxing Fan"/><link rel="author" href="https://computationalrobotics.seas.harvard.edu/"/><meta name="author" content="Heng Yang"/><meta name="keywords" content="SceneAgent,real-to-sim,sim-to-real,3D Gaussian Splatting,simulation environments,predictive physics,robot policy evaluation,policy training,online planning,object segmentation,articulation,deformable objects,digital sisters,robot learning,Harvard Computational Robotics Group"/><meta name="creator" content="Harvard Computational Robotics Group"/><meta name="publisher" content="Harvard Computational Robotics Group"/><meta name="robots" content="index, follow"/><meta name="googlebot" content="index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1"/><meta name="citation_title" content="SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments"/><meta name="citation_author" content="Luke Hollis"/><meta name="citation_author" content="Tianxing Fan"/><meta name="citation_author" content="Heng Yang"/><meta name="citation_publication_date" content="2026"/><meta name="citation_publisher" content="Harvard Computational Robotics Group"/><meta name="citation_abstract_html_url" content="https://computationalrobotics.seas.harvard.edu/SceneAgent/"/><link rel="canonical" href="https://computationalrobotics.seas.harvard.edu/SceneAgent/"/><meta property="og:title" content="SceneAgent: 3D captures to simulation environments with predictive physics"/><meta property="og:description" content="An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint."/><meta property="og:url" content="https://computationalrobotics.seas.harvard.edu/SceneAgent/"/><meta property="og:site_name" content="Harvard Computational Robotics Group"/><meta property="og:locale" content="en_US"/><meta property="og:image" content="https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"/><meta property="og:image:secure_url" content="https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"/><meta property="og:image:type" content="image/jpeg"/><meta property="og:image:width" content="1200"/><meta property="og:image:height" content="630"/><meta property="og:image:alt" content="SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle"/><meta property="og:type" content="website"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:title" content="SceneAgent: 3D captures to simulation environments with predictive physics"/><meta name="twitter:description" content="An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint."/><meta name="twitter:image" content="https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg"/><meta name="twitter:image:alt" content="SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle"/><link rel="icon" href="./favicon.png" type="image/png"/><script src="./_next/static/chunks/0cz1d0mv5g_q7.js" noModule=""></script></head><body><div hidden=""><!--$--><!--/$--></div><div class="shell"><nav class="rail" aria-label="Page progress"><div class="rail__track" style="--n:8;--k:0"><div class="rail__line" aria-hidden="true"></div><div class="rail__fill" aria-hidden="true"></div><button type="button" class="rail__item" aria-current="true" title="Intro"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Intro</span></button><button type="button" class="rail__item" aria-current="false" title="Pipeline"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Pipeline</span></button><button type="button" class="rail__item" aria-current="false" title="Digital twins and sisters"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Digital twins and sisters</span></button><button type="button" class="rail__item" aria-current="false" title="Hybrid scenes"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Hybrid scenes</span></button><button type="button" class="rail__item" aria-current="false" title="Agents"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Agents</span></button><button type="button" class="rail__item" aria-current="false" title="Policy training"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Policy training</span></button><button type="button" class="rail__item" aria-current="false" title="Evaluation"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Evaluation</span></button><button type="button" class="rail__item" aria-current="false" title="Citation"><span class="rail__dot" aria-hidden="true"></span><span class="rail__label px">Citation</span></button></div></nav><main><section id="top" style="background:var(--paper);padding:40px var(--chapter-pad) 56px"><div style="max-width:calc(var(--page-w) - 2 * var(--chapter-pad))"><div class="cover__brand"><a class="lab-logo" href="https://computationalrobotics.seas.harvard.edu/"><img src="./branding/computational-robotics.jpg" alt="Harvard Computational Robotics Group" width="2286" height="299"/></a></div><div style="border-top:3px solid var(--ink);padding-top:10px;display:flex;justify-content:space-between;font-size:13px;line-height:1.45;color:var(--ink)"><div>Harvard Computational Robotics Group<br/>School of Engineering and Applied Sciences, Harvard University</div><div style="text-align:right">Preprint<br/>2026</div></div><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden;margin-top:40px"><video src="./videos/cover_video.mp4" poster="./videos/cover_video.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="SceneAgent cover video" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div></figure><h1 class="cover__title">SceneAgent</h1><div style="margin-top:12px;max-width:640px;font-size:14px;color:var(--text-muted)">An agentic pipeline for converting 3D captures to simulation environments with predictive physics for policy evaluation, online planning, and policy training</div><div style="margin-top:24px;font-size:15px;font-weight:700;color:var(--ink)"><span><a href="mailto:lhollis@g.harvard.edu">Luke Hollis</a><sup>1<!-- -->*</sup>, </span><span><a href="mailto:tianxing_fan@seas.harvard.edu">Tianxing Fan</a><sup>1<!-- -->*</sup>, </span><span><a href="mailto:hankyang@seas.harvard.edu">Heng Yang</a><sup>1</sup></span></div><div style="margin-top:8px;font-size:13px;line-height:1.5;color:var(--text-muted)"><span><sup>1</sup> <a href="https://computationalrobotics.seas.harvard.edu/" target="_blank" rel="noopener">Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University</a></span><br/><sup>*</sup> <!-- -->Equal contribution</div><div style="margin-top:24px;display:flex;align-items:center;gap:16px;flex-wrap:wrap"><button type="button" disabled="" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:default;opacity:0.4;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:var(--accent);color:var(--text-on-accent);text-transform:uppercase">Paper</button><button type="button" disabled="" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:default;opacity:0.4;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink);text-transform:uppercase">Code</button><span style="font-size:12px;text-transform:uppercase;color:var(--text-muted)">[coming soon]</span></div><p class="chapter__text">SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS.</p></div></section><section class="chapter chapter--bleed" id="pipeline"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">1</div><div><h2 class="chapter__title">From Real-world Scenes into Simulation</h2><p class="chapter__lead">Our pipeline supports converting 3D capture data from 3DGS, photogrammetry, and LiDAR into usable scenes that can be used across a wide range of simulators, such as Isaac Lab, MuJoCo, and Unreal Engine.</p></div></div><div class="chapter__text"><p>We initially infer semantic features for each 3d Gaussian and save them in a codebook for rapid lookup, inspired by the method from LangSplat. We then segment the foreground objects in the scene from the background and crop their 3d Gaussians from the background. We then predictively infill these cropped regions. For each object, we predict a physics material with a combination of semantic features, inference against a per-gaussian predictive physics model, and use VLM to estimate features such as friction coefficient, rigidity, mass, and density.</p><p>From this data, we then bake a physics material for each object and then decompose that object into smaller pieces. We then add articulations to the objects as necessary so they can perform actions as they would in real life. Finally, we generate versions of each object with slight variations–instead of digital twins, we term these digital sisters. We then use a VLM to inspect and review each of these steps as well as the transform of the converted objects in the scene.</p></div><div class="chapter__exhibit chapter__exhibit--bleed" data-bleed=""><div class="pipeline-selector" role="group" aria-label="Choose a scene for the diagram and viewer"><button type="button" aria-pressed="true" aria-label="01 Greenhouse Flower (Rudbeckia hirta)" aria-controls="pipeline-scene-diagram pipeline-scene-viewer"><span class="pipeline-selector__image"><img src="./pipeline/garden/preview.webp" alt=""/></span><span class="pipeline-selector__label"><b>01</b><strong>Greenhouse</strong></span></button><button type="button" aria-pressed="false" aria-label="02 Computational Robotics Lab Sony radio cassette player" aria-controls="pipeline-scene-diagram pipeline-scene-viewer"><span class="pipeline-selector__image"><img src="./pipeline/lab/preview.webp" alt=""/></span><span class="pipeline-selector__label"><b>02</b><strong>Computational Robotics Lab</strong></span></button><button type="button" aria-pressed="false" aria-label="03 Kitchen Pink teacup" aria-controls="pipeline-scene-diagram pipeline-scene-viewer"><span class="pipeline-selector__image"><img src="./pipeline/kitchen2/preview.webp" alt=""/></span><span class="pipeline-selector__label"><b>03</b><strong>Kitchen</strong></span></button><button type="button" aria-pressed="false" aria-label="04 World Labs Kitchen (Generated 3DGS · Marble) Hawaiian pizza" aria-controls="pipeline-scene-diagram pipeline-scene-viewer"><span class="pipeline-selector__image"><img src="./pipeline/worldlabs/preview.webp" alt=""/></span><span class="pipeline-selector__label"><b>04</b><strong>World Labs Kitchen</strong></span><span class="pipeline-selector__note">Generated 3DGS · Marble</span></button><button type="button" aria-pressed="false" aria-label="05 Observatory (Matterport · Coming soon) Telescope" disabled="" aria-controls="pipeline-scene-diagram pipeline-scene-viewer"><span class="pipeline-selector__image"><img src="./pipeline/observatory/preview.webp" alt=""/></span><span class="pipeline-selector__label"><b>05</b><strong>Observatory</strong></span><span class="pipeline-selector__note">Matterport · Coming soon</span></button></div><div id="pipeline-scene-diagram"><div class="pipeline-diagram" data-scene="garden" aria-label="Greenhouse: Flower (Rudbeckia hirta) through SceneAgent's six steps"><canvas aria-hidden="true"></canvas><div class="pipeline-input"><h3>Flower (Rudbeckia hirta)</h3><div class="pipeline-capture" role="img" aria-label="Flower (Rudbeckia hirta) reconstructed at the centre of three image planes and camera frustums"><div class="pipeline-viewport pipeline-capture__space" aria-hidden="true"></div></div><p class="pipeline-prompt" aria-label="Prompt: convert the flower 3d capture into a usable simulation asset for Isaac Lab"><span aria-hidden="true">> </span><span class="pipeline-prompt__text"></span><span class="pipeline-prompt__cursor" data-done="false" aria-hidden="true"></span></p></div><div class="pipeline-arrow" aria-hidden="true">→</div><section class="scene-agent" aria-labelledby="scene-agent-title"><div class="scene-agent__header"><h3 id="scene-agent-title">SceneAgent</h3><p>Processes each 3D object for predictive physics, decomposition, and object properties</p></div><div class="scene-agent__grid"><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>01</span>3DGS</h4><p>Convert input RGB or other 3D data</p></div><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>02</span>Semantic features</h4><p>Use GroundingDINO + SAM to assign semantic features to each 3D Gaussian</p></div><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>03</span>Predictive physics</h4><p>Generate predictive per-Gaussian physics priors to bake physics material</p></div><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>04</span>Object decomposition</h4><p>Decompose each object into individual parts as needed</p></div><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>05</span>Articulation</h4><p>Articulate any movable parts within each object</p></div><div class="scene-agent__cell"><div class="pipeline-viewport " aria-hidden="true"></div><h4><span>06</span>Object sisters</h4><p>Generated object variations with geometric or visual differences</p></div></div></section><div class="pipeline-arrow" aria-hidden="true">→</div><div class="pipeline-output"><div role="img" aria-label="Greenhouse: complete scene objects with Panda + Sharpa Wave hand, without a background"><div class="pipeline-viewport pipeline-output__scene" aria-hidden="true"></div></div><h3><span>07</span> Package scene in Universal Scene Description format</h3><p>Share usable variations of the full scene as USDZ for NVIDIA Isaac Lab, MuJoCo, Unreal Engine, and other simulators.</p></div></div></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">1.2</div></div></div><div class="chapter__exhibit chapter__exhibit--bleed sub" data-bleed=""><h3 class="pipeline-step"><span>08</span> Interactive viewer</h3><div id="pipeline-scene-viewer"><div class="scene-playback" data-scene="garden"><button type="button" class="stage scene-hold"><img src="./pipeline/garden/preview.webp" alt="Greenhouse: Panda + Sharpa Wave hand"/><span class="scene-hold__control"><span class="scene-hold__play"><i aria-hidden="true"></i>Play the scene</span><span class="scene-hold__note">Tap to start the viewer. This may crash on mobile devices.</span></span></button></div></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--ink)">Figure 1 Pipeline architecture. We support converting input RGB image sequences (from video or as images), pre-existing 3DGS or photogrammetry assets, and LiDAR scans. Output is in Universal Scene Description format (USDZ) and able to be used in a wide range of simulation tools such as Isaac Lab, MuJoCo, and Unreal Engine.</div><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">1.3</div></div></div><div class="chapter__text"><p>After we convert the scene, we created an interactive viewer built with<!-- --> <a href="https://threejs.org/" target="_blank" rel="noopener">Three.js</a> <!-- -->to assist in positioning the robot(s) as needed within the scene and making further edits. This allows for adding objects from NVIDIA Omniverse or further cropping and modifying the background 3DGS as well as fixing any imperfections with conversion.</p><p>The final converted scene is exported in Universal Scene Description format so that it can be used in modern simulation environments, such as Isaac Lab, MuJoCo, and Unreal Engine. We tested on the Franka Emika Panda, following a modified DROID setup with the Universal Manipulation Interface gripper, so our featured simulation examples use this setup.</p></div></section><section class="chapter" id="cousins"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">2</div><div><h2 class="chapter__title">Digital Twins and Sisters</h2><p class="chapter__lead">In order to increase training generalizability from simulation back to the real world, we create slight variations of each object in the scenes converted by the pipeline.</p></div></div><div class="chapter__exhibit"><div class="cousin-rails"><div class="cousin-gallery"><div class="cousin-gallery__toolbar"><div><b>Flower (Rudbeckia hirta)</b><span class="px"> <!-- -->9<!-- --> sisters</span></div><div class="cousin-gallery__actions"><span class="px" aria-live="polite">Loading 0 / 10</span><button type="button" aria-label="Previous sisters of Flower (Rudbeckia hirta)" disabled="" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:default;opacity:0.4;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">←</button><button type="button" aria-label="Next sisters of Flower (Rudbeckia hirta)" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:pointer;opacity:1;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">→</button></div></div><div class="cousin-gallery__viewport" tabindex="0" aria-label="Horizontally scrolling sisters of Flower (Rudbeckia hirta)"><div class="cousin-gallery__track"><canvas aria-hidden="true"></canvas><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>00</b><span>original</span></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>01</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>02</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>03</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>04</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>05</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>06</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>07</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>08</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>09</b></div></div></article></div></div></div><div class="cousin-gallery"><div class="cousin-gallery__toolbar"><div><b>Sony CFS-43</b><span class="px"> <!-- -->9<!-- --> sisters</span></div><div class="cousin-gallery__actions"><span class="px" aria-live="polite">Loading 0 / 10</span><button type="button" aria-label="Previous sisters of Sony CFS-43" disabled="" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:default;opacity:0.4;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">←</button><button type="button" aria-label="Next sisters of Sony CFS-43" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:pointer;opacity:1;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">→</button></div></div><div class="cousin-gallery__viewport" tabindex="0" aria-label="Horizontally scrolling sisters of Sony CFS-43"><div class="cousin-gallery__track"><canvas aria-hidden="true"></canvas><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>00</b><span>original</span></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>01</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>02</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>03</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>04</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>05</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>06</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>07</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>08</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>09</b></div></div></article></div></div></div><div class="cousin-gallery"><div class="cousin-gallery__toolbar"><div><b>Pink teacup</b><span class="px"> <!-- -->9<!-- --> sisters</span></div><div class="cousin-gallery__actions"><span class="px" aria-live="polite">Loading 0 / 10</span><button type="button" aria-label="Previous sisters of Pink teacup" disabled="" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:default;opacity:0.4;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">←</button><button type="button" aria-label="Next sisters of Pink teacup" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:pointer;opacity:1;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">→</button></div></div><div class="cousin-gallery__viewport" tabindex="0" aria-label="Horizontally scrolling sisters of Pink teacup"><div class="cousin-gallery__track"><canvas aria-hidden="true"></canvas><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>00</b><span>original</span></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>01</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>02</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>03</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>04</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>05</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>06</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>07</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>08</b></div></div></article><article class="cousin-family"><div class="cousin-family__view" aria-hidden="true"></div><div class="cousin-family__cap"><div class="cousin-family__index"><b>09</b></div></div></article></div></div></div><div class="cousin-rails__more"><button type="button" aria-expanded="false" style="font-family:var(--font-sans);font-weight:700;font-size:13px;padding:6px 14px;border:1px solid transparent;cursor:pointer;opacity:1;transition:background 120ms, color 120ms;line-height:1.2;display:inline-flex;align-items:center;gap:8px;text-decoration:none;background:transparent;color:var(--ink);border-color:var(--ink)">Show more</button></div></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">2.2</div></div></div><div class="chapter__text"><p>Instead of digital twins, these are rather “digital sisters,” not exact replicas of reality but similar variations that reflect the range of diversity that a manipulator may encounter in real-world scenarios. We also randomize placement of objects and lighting to continue to improve policy performance in real-world rollouts.</p><p>In our initial testing, this increases real-world task completion rates after training in our converted simulation digital sister scenes, but we are continuing to test and evaluate how different lighting, placement, and generative 3d models influence policy performance from simulation back to the real world.</p><p>We chose to randomize each of these factors when composing full digital sister scenes, but future work could test each in isolation to verify its influence on policy generalizability during finetuning to study which is best to spend more iterations on.</p></div><div class="chapter__exhibit sub"><div class="clips" style="grid-template-columns:repeat(2, minmax(0, 1fr))"><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/warehouse_suitcase_suitcase.mp4" poster="./videos/warehouse_suitcase_suitcase.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Warehouse · a different suitcase" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Contact rich tasks with predictive physics, such as rolling a red rubber ball from one side of the suitcase to the other</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/studio_flower_noir_cinema.mp4" poster="./videos/studio_flower_noir_cinema.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · flower grasp, relit" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Picking up delicate items such as the flower 3d capture from the previous greenhouse scene in hybrid digital art and 3d capture environments</div></figcaption></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">2.3</div></div></div><div class="chapter__text"><p>Along with the typical tasks being trained in simulation policy, we are seeking to understand better how the predictive physics properties can help us achieve a series of contact-rich tasks, such as rolling a red rubber ball from one side of the suitcase to another with a gripper on the Franka arm or picking a columnar object from a tightly packed box.</p><p>In future iterations, we also hope to study further how small perturbations of the physics properties may also influence training in simulation. For example, if an object’s mass, friction coefficient, or rigidity are varied slightly between versions, will this help our policy during finetuning more accurately grasp and manipulate deformable objects or objects with heterogeneous physics properties (such as shoes, hats, cardboard boxes, and similar).</p></div></section><section class="chapter chapter--bleed" id="hybrid"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">3</div><div><h2 class="chapter__title">Hybrid Scenes with 3d Captures and Digital Art</h2><p class="chapter__lead">We convert individual 3d captured objects into simulation assets with our SceneAgent pipeline and incorporate them into synthetic scenes prepared by 3d artists.</p></div></div><div class="chapter__exhibit chapter__exhibit--bleed" data-bleed=""><div class="clips" style="grid-template-columns:repeat(3, minmax(0, 1fr))"><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/kitchen_apple.mp4" poster="./videos/kitchen_apple.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Kitchen · apple" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Hybrid scenes from available assets in Isaac Lab, a Kitchen scene picking fruit, similar to RoboCasa</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/warehouse_pack.mp4" poster="./videos/warehouse_pack.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Warehouse · pack" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Contact rich tasks picking tightly packed vertical cylinders</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/chem_lab_sort.mp4" poster="./videos/chem_lab_sort.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Chemistry lab · sort" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Chemistry lab, sorting vials into beakers</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/datacenter_cable.mp4" poster="./videos/datacenter_cable.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Data center · cable" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Picking and plugging cables into a data center server rack</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/studio_cardboard_fold.mp4" poster="./videos/studio_cardboard_fold.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · cardboard fold" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Two arms folding a cardboard packaging in simulation, packing a deformable object</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/studio_succulent_watering.mp4" poster="./videos/studio_succulent_watering.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · succulent watering" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Hybrid digital art and 3d capture scene with a Franka Emika Panda arm watering a succulent with an eyedropper</div></figcaption></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">3.2</div></div></div><div class="chapter__text"><p>Because many backgrounds and environments are redundant across common tasks, we combine specific 3d captured objects processed with our SceneAgent pipeline with other existing art assets created by 3d artists, such as in the NVIDIA Omniverse asset repository. This allows for rapid development of common scenes for policy evaluation along with scenes with specific objects for individual tasks.</p><p>For example, when a manipulator may need to learn how to use a specific part, say for a new healthcare device or a new cooling fan in a server rack, that specific part individually could be 3d captured and processed in our pipeline and then incorporated into a pre-existing hospital or data center scene such as the default environments that accompany NVIDIA Isaac Lab.</p></div><div class="chapter__exhibit chapter__exhibit--bleed sub" data-bleed=""><div class="clips" style="grid-template-columns:repeat(3, minmax(0, 1fr))"><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/sharpa_hand_flower_scoop_realtime.mp4" poster="./videos/sharpa_hand_flower_scoop_realtime.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · Sharpa hand, real time" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A Sharpa dexterous hand scooping to cup a delicate flower in its palm</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/hands_clipping_in_gpu.mp4" poster="./videos/hands_clipping_in_gpu.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Dexterous hands · GPU installation" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Two dextrous Sharpa hands clipping a GPU into place into a motherboard</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/hands_sunglasses_5x.mp4" poster="./videos/hands_sunglasses_5x.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Dexterous hands · sunglasses, 5× speed" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Sharpa hand picking up and unfolding sunglasses then putting them on a face</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/dexmate_sharpa_pizza_and_dishes.mp4" poster="./videos/dexmate_sharpa_pizza_and_dishes.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Kitchen · Dexmate dual Sharpa" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A Dexmate mobile robot with dual Sharpa dexterous hands serving a pizza slice onto a plate and clearing the dishes to the sink</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/studio_sunglasses_noir.mp4" poster="./videos/studio_sunglasses_noir.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · Sharpa sunglasses, noir" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Two Sharpa dexterous hands lifting and unfolding a pair of sunglasses, then setting them on a bust</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/sharpa_hello_world_lift_then_uncap.mp4" poster="./videos/sharpa_hello_world_lift_then_uncap.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Studio · Sharpa lift and uncap" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Dual robot arms lifting a marker into a Sharpa dexterous hand, uncapping it, and writing Hello World on a whiteboard</div></figcaption></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">3.3</div></div></div><div class="chapter__text"><p>The generated environments can be adapted for policy evaluation and training for multiple embodiments, and we believe that they will be especially useful for training dextrous hands. Whereas manipulators like the UMI gripper on the Franka Emika Panda can benefit from more accurate physics properties on objects in simulation, dextrous hands must learn to accomplish much more complex tasks with deformable fabrics, plastics, sticky surfaces, and similar challenges. We hope in future simulations to have a pipeline with increased accuracy around specifically supporting packing in plastic bags and applying stickers to diverse surfaces.</p><p>Alternative methods of course may use generative diffusion- or autogressive-based world models to generate the multi-view video used for policy evaluation and finetuning. Our method may be beneficial because it is much less computationally intensive than methods relying on generative video models and can be precise for specific environments or parts.</p></div></section><section class="chapter chapter--bleed" id="agents"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">4</div><div><h2 class="chapter__title">Agentic Pipeline for SceneAgent</h2><p class="chapter__lead">We provide a team of agents to convert many types of 3d captures into fully usable simulation scenes ready for training in Isaac Lab, MuJoCo, or Unreal. The easiest way to convert a 3d capture is by an agent swarm that processes each step of the pipeline and visually inspects and reviews it for errors.</p></div></div><div class="chapter__exhibit chapter__exhibit--bleed" data-bleed=""><div class="spread" role="img" aria-label="The agentic workflow: nine agents in three stages, Input, Conversion and Sisters, processing 3d capture data into usable simulation scenes."><div class="spread__grid" style="--spread-fluid:minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr);--spread-fixed:calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s))"><svg class="spread__col" viewBox="0 0 480 932" style="font-family:Helvetica, "Helvetica Neue", Arimo, Arial, sans-serif"><g><text x="0" y="26" font-size="26" font-weight="700" fill="var(--accent)">00</text><text x="46" y="26" font-size="26" font-weight="700" fill="var(--ink)">Input</text><line x1="0" y1="38.5" x2="480" y2="38.5" stroke="var(--ink)"></line><text x="0" y="64" font-size="24" fill="var(--text-muted)">Process input 3d data from a diverse</text><text x="0" y="92" font-size="24" fill="var(--text-muted)">range of sources, including RGB images,</text><text x="0" y="120" font-size="24" fill="var(--text-muted)">pre-processed photogrammetry files,</text><text x="0" y="148" font-size="24" fill="var(--text-muted)">or LiDAR</text></g><g><g class="agent-node" role="img" aria-label="01 Orchestrator: Plans the full conversion pipeline based on the status of the input data"><rect x="0.5" y="168.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 184) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:0"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="6" y="0" width="1" height="1"></rect><rect x="6" y="1" width="1" height="1"></rect><rect x="6" y="2" width="1" height="1"></rect></g></g></g><text x="129" y="210" font-size="26" font-weight="700" fill="var(--accent)">01</text><text x="173" y="210" font-size="26" font-weight="700" fill="var(--ink)">Orchestrator</text><text x="14" y="265" font-size="21" fill="var(--text-body)">Plans the full conversion pipeline based</text><text x="14" y="291" font-size="21" fill="var(--text-body)">on the status of the input data</text><text x="14" y="320" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-RUN-PIPELINE</text></g><line x1="240" y1="334" x2="240" y2="362" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="02 Ingest: Process input RGB or other data sources to 3DGS"><rect x="0.5" y="362.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 378) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:1"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="2" y="0" width="9" height="1"></rect><rect x="4" y="1" width="5" height="1"></rect></g></g></g><text x="129" y="404" font-size="26" font-weight="700" fill="var(--accent)">02</text><text x="173" y="404" font-size="26" font-weight="700" fill="var(--ink)">Ingest</text><text x="14" y="459" font-size="21" fill="var(--text-body)">Process input RGB or other data sources</text><text x="14" y="485" font-size="21" fill="var(--text-body)">to 3DGS</text><text x="14" y="514" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-INGEST-DATA</text></g><line x1="240" y1="528" x2="240" y2="556" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="03 Step runner: Process the tools for pipeline with visual review of the results, including object masks, semantic features, physics properties, background crop and infill, etc"><rect x="0.5" y="556.5" width="479" height="217" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 572) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:2"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="4" y="0" width="1" height="1"></rect><rect x="6" y="0" width="1" height="1"></rect><rect x="8" y="0" width="1" height="1"></rect><rect x="4" y="1" width="1" height="1"></rect><rect x="6" y="1" width="1" height="1"></rect><rect x="8" y="1" width="1" height="1"></rect></g></g></g><text x="129" y="598" font-size="26" font-weight="700" fill="var(--accent)">03</text><text x="173" y="598" font-size="26" font-weight="700" fill="var(--ink)">Step runner</text><text x="14" y="653" font-size="21" fill="var(--text-body)">Process the tools for pipeline with visual</text><text x="14" y="679" font-size="21" fill="var(--text-body)">review of the results, including object</text><text x="14" y="705" font-size="21" fill="var(--text-body)">masks, semantic features, physics</text><text x="14" y="731" font-size="21" fill="var(--text-body)">properties, background crop and infill, etc</text><text x="14" y="760" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-RUN-STEP</text></g></g></svg><div class="spread__gutter" aria-hidden="true"><svg viewBox="0 0 144 932" preserveAspectRatio="none"><polyline points="0,665 72,665 72,251 144,251" fill="none" stroke="var(--text-label)" vector-effect="non-scaling-stroke"></polyline></svg></div><svg class="spread__col" viewBox="0 0 480 932" style="font-family:Helvetica, "Helvetica Neue", Arimo, Arial, sans-serif"><g><text x="0" y="26" font-size="26" font-weight="700" fill="var(--accent)">01</text><text x="46" y="26" font-size="26" font-weight="700" fill="var(--ink)">Conversion</text><line x1="0" y1="38.5" x2="480" y2="38.5" stroke="var(--ink)"></line><text x="0" y="64" font-size="24" fill="var(--text-muted)">Segment and predict properties for</text><text x="0" y="92" font-size="24" fill="var(--text-muted)">objects in scan, then package and inspect</text></g><g><g class="agent-node" role="img" aria-label="04 Alignment: Scale, place and rotate each asset by photometric, silhouette and edge residuals"><rect x="0.5" y="168.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 184) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:3"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="2" y="0" width="1" height="1"></rect><rect x="4" y="0" width="5" height="1"></rect><rect x="10" y="0" width="1" height="1"></rect></g></g></g><text x="129" y="210" font-size="26" font-weight="700" fill="var(--accent)">04</text><text x="173" y="210" font-size="26" font-weight="700" fill="var(--ink)">Alignment</text><text x="14" y="265" font-size="21" fill="var(--text-body)">Scale, place and rotate each asset by</text><text x="14" y="291" font-size="21" fill="var(--text-body)">photometric, silhouette and edge residuals</text><text x="14" y="320" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-ALIGN-SCENE</text></g><line x1="240" y1="334" x2="240" y2="362" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="05 Orientation: Isaac Lab sweeps that orient the completed background 3DGS to the exported scene"><rect x="0.5" y="362.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 378) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:4"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="5" y="0" width="3" height="1"></rect><rect x="6" y="1" width="1" height="1"></rect></g></g></g><text x="129" y="404" font-size="26" font-weight="700" fill="var(--accent)">05</text><text x="173" y="404" font-size="26" font-weight="700" fill="var(--ink)">Orientation</text><text x="14" y="459" font-size="21" fill="var(--text-body)">Isaac Lab sweeps that orient the completed</text><text x="14" y="485" font-size="21" fill="var(--text-body)">background 3DGS to the exported scene</text><text x="14" y="514" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-ORIENT-BACKGROUND</text></g><line x1="240" y1="528" x2="240" y2="556" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="06 Packager: Publishes the data_processed package for the editor as USDZ"><rect x="0.5" y="556.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 572) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:5"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="4" y="0" width="5" height="1"></rect><rect x="4" y="1" width="5" height="1"></rect></g></g></g><text x="129" y="598" font-size="26" font-weight="700" fill="var(--accent)">06</text><text x="173" y="598" font-size="26" font-weight="700" fill="var(--ink)">Packager</text><text x="14" y="653" font-size="21" fill="var(--text-body)">Publishes the data_processed package for</text><text x="14" y="679" font-size="21" fill="var(--text-body)">the editor as USDZ</text><text x="14" y="708" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-PACKAGE-DATA-PROCESSED</text></g><line x1="240" y1="722" x2="240" y2="750" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="07 Inspector: Visually reviews and fixes anything off in the final prepared USDZ in NVIDIA Isaac Lab"><rect x="0.5" y="750.5" width="479" height="165" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 766) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:6"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="4" y="0" width="4" height="1"></rect><rect x="4" y="1" width="2" height="1"></rect></g></g></g><text x="129" y="792" font-size="26" font-weight="700" fill="var(--accent)">07</text><text x="173" y="792" font-size="26" font-weight="700" fill="var(--ink)">Inspector</text><text x="14" y="847" font-size="21" fill="var(--text-body)">Visually reviews and fixes anything off in</text><text x="14" y="873" font-size="21" fill="var(--text-body)">the final prepared USDZ in NVIDIA Isaac Lab</text><text x="14" y="902" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-INSPECT-ISAAC</text></g></g></svg><div class="spread__gutter" aria-hidden="true"><svg viewBox="0 0 144 932" preserveAspectRatio="none"><polyline points="0,833 72,833 72,264 144,264" fill="none" stroke="var(--text-label)" vector-effect="non-scaling-stroke"></polyline></svg></div><svg class="spread__col" viewBox="0 0 480 932" style="font-family:Helvetica, "Helvetica Neue", Arimo, Arial, sans-serif"><g><text x="0" y="26" font-size="26" font-weight="700" fill="var(--accent)">02</text><text x="46" y="26" font-size="26" font-weight="700" fill="var(--ink)">Sisters</text><line x1="0" y1="38.5" x2="480" y2="38.5" stroke="var(--ink)"></line><text x="0" y="64" font-size="24" fill="var(--text-muted)">From the fully processed digital twin,</text><text x="0" y="92" font-size="24" fill="var(--text-muted)">generate a family of sisters</text></g><g><g class="agent-node" role="img" aria-label="08 Object sisters: Generate small variations of each object in the scene with generative 3d models, focusing on geometric and visual differences"><rect x="0.5" y="168.5" width="479" height="191" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 184) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:7"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="3" y="0" width="2" height="1"></rect><rect x="8" y="0" width="2" height="1"></rect><rect x="3" y="1" width="2" height="1"></rect><rect x="8" y="1" width="2" height="1"></rect></g></g></g><text x="129" y="210" font-size="26" font-weight="700" fill="var(--accent)">08</text><text x="173" y="210" font-size="26" font-weight="700" fill="var(--ink)">Object sisters</text><text x="14" y="265" font-size="21" fill="var(--text-body)">Generate small variations of each object</text><text x="14" y="291" font-size="21" fill="var(--text-body)">in the scene with generative 3d models,</text><text x="14" y="317" font-size="21" fill="var(--text-body)">focusing on geometric and visual differences</text><text x="14" y="346" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-OBJECT-SISTERS</text></g><line x1="240" y1="360" x2="240" y2="388" stroke="var(--text-label)"></line></g><g><g class="agent-node" role="img" aria-label="09 Scene sisters: Generate different layouts and arrangement of the sister objects and export them as separate USDZ scenes"><rect x="0.5" y="388.5" width="479" height="191" fill="var(--paper)" stroke="var(--ink)"></rect><g transform="translate(14 404) scale(7)" fill="currentColor" shape-rendering="crispEdges" aria-hidden="true"><g class="agent-sprite" style="--i:8"><g class="agent-sprite__leg"><rect x="4" y="6" width="1" height="1"></rect><rect x="6" y="6" width="1" height="1"></rect><rect x="8" y="6" width="1" height="1"></rect></g><g class="agent-sprite__body"><rect x="2" y="3" width="9" height="1"></rect><rect x="2" y="4" width="2" height="1"></rect><rect x="5" y="4" width="3" height="1"></rect><rect x="9" y="4" width="2" height="1"></rect><rect x="2" y="5" width="9" height="1"></rect></g><g class="agent-sprite__tool"><rect x="3" y="0" width="1" height="1"></rect><rect x="9" y="0" width="1" height="1"></rect><rect x="2" y="1" width="9" height="1"></rect></g></g></g><text x="129" y="430" font-size="26" font-weight="700" fill="var(--accent)">09</text><text x="173" y="430" font-size="26" font-weight="700" fill="var(--ink)">Scene sisters</text><text x="14" y="485" font-size="21" fill="var(--text-body)">Generate different layouts and arrangement</text><text x="14" y="511" font-size="21" fill="var(--text-body)">of the sister objects and export them as</text><text x="14" y="537" font-size="21" fill="var(--text-body)">separate USDZ scenes</text><text x="14" y="566" font-size="15" font-family="var(--font-pixel)" letter-spacing="0.08em" fill="var(--text-label)">REAL2SIM-SCENE-SISTERS</text></g></g></svg></div></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--ink)">Figure 4 · An overview of the full agentic pipeline with various tools and skills for processing 3d capture data into usable simulation scenes</div><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">4.2</div></div></div><div class="chapter__text"><p>Our agentic workflow processes 3D capture data of diverse origins into fully usable simulation scenes in Isaac Lab, MuJoCo, or Unreal Engine. The agents are helpful in this case specifically because of the diversity of data formats of 3d data and error-prone nature of object conversion and decomposition models. For example, a common failure mode is that the 3d object segmentation will incorrectly transform the scale, rotation, or placement of the objects in the scene, and agents can take camera renders from within Isaac Lab to visually inspect the conversion results and rapidly correct such small errors.</p></div><div class="chapter__exhibit chapter__exhibit--bleed" data-bleed=""><div class="clips" style="grid-template-columns:repeat(3, minmax(0, 1fr))"><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/objects/sony_radio_clean.mp4" poster="./videos/objects/sony_radio_clean.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Sony radio · full control articulation" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Sony CFS-43, photogrammetry of a radio cassette player and articulations of its door, keys, knobs, and selector</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/stegosaurus_walking.mp4" poster="./videos/stegosaurus_walking.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Stegosaurus · walking" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A 3dgs of a stegosaurus skeleton from the AMNH, fully articulated and walking (without hand-created animations)</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/objects/ge90_engine_orbit_zoom.mp4" poster="./videos/objects/ge90_engine_orbit_zoom.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Jet engine · orbit and detail zoom" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A jet engine with processed turbofan, articulated and running based on engine schematics</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/objects/sceneagent_bencini_camera.mp4" poster="./videos/objects/sceneagent_bencini_camera.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Bencini cine camera · orbit and detail zoom" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A Bencini cine camera converted into a usable simulation asset, with usable frame-rate dial and battery compartment</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/objects/sceneagent_cessna_cockpit.mp4" poster="./videos/objects/sceneagent_cessna_cockpit.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Cessna cockpit · orbit and detail zoom" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A Cessna cockpit, converted into simulation asset with extreme detail in its instrument panel, switches, and placards</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:4 / 3;overflow:hidden"><video src="./videos/objects/sceneagent_motorcycle.mp4" poster="./videos/objects/sceneagent_motorcycle.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Motorcycle · orbit and detail zoom" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A motorcycle converted from 3d capture into a usable simulation asset, including many details</div></figcaption></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">4.3</div></div></div><div class="chapter__text"><p>One of the most unique and beneficial aspects about our agentic workflow is that it can support converting multiple different data types from diverse sources with useful results. For example, it can similarly process input RGB images and align them with structure-from-motion with colmap and then train a 3DGS scene with NerfStudio’s <code>splatfacto</code> big and inspect, review, and correct any results, such as the per image color calibration of the input images. The same pipeline will equally process public internet 3d capture data from photogrammetry or pre-existing 3DGS .ply files.</p></div><div class="chapter__exhibit chapter__exhibit--bleed" data-bleed=""><div class="clips" style="grid-template-columns:repeat(3, minmax(0, 1fr))"><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/deformables_textile_rack_orbit.mp4" poster="./videos/deformables_textile_rack_orbit.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Textile rack · orbit" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A patterned textile draped over a garment rack, orbited to read the folds from every side</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/deformables_hoodie_lift_drape.mp4" poster="./videos/deformables_hoodie_lift_drape.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Hoodie · lift and drape" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A hoodie lifted from a surface and left to drape under gravity</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/deformables_towel_rack_orbit.mp4" poster="./videos/deformables_towel_rack_orbit.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Towel rack · orbit" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">Three towels hung on a wall rack, orbited to read the folds from every side</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/deformables_firefighter_hose_unwind.mp4" poster="./videos/deformables_firefighter_hose_unwind.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Firefighter hose · unwind" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A coiled firefighter hose unwinding under soft-body dynamics</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/liquid_death_hat_physics.mp4" poster="./videos/liquid_death_hat_physics.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Liquid Death hat · physics" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">A Liquid Death corduroy cap with soft-body physics, deforming under contact dynamics</div></figcaption></figure><figure style="margin:0;min-width:0"><div style="position:relative;border:1px solid var(--hairline);background:var(--ink);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/deformables_extension_cord_unwind.mp4" poster="./videos/deformables_extension_cord_unwind.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Extension cord · unwind" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div><figcaption style="margin-top:10px"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--text-muted)">An extension cord uncoiling from its wound state</div></figcaption></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">4.4</div></div></div><div class="chapter__text"><p>Likewise, it is able to turn synthetic datasets from generative 3DGS models (such as World Labs Marble, NVIDIA Lyra, or TenCent Hy-World). One can prompt the pipeline to convert the full scene into segmented 3d objects with predictive physics properties or instead focus only foreground objects. The first is much more time-consuming and error prone at this point, but we believe that future iterations of the pipeline will easily improve this.</p></div><div class="chapter__text"><p>We imagine a future application of this agentic workflow running on individual in-home or in-factory robots that can 3d capture a space, process that space on-device only with an NVIDIA Jetson Orin or Thor or equivalent, and then train a finetune of its policy on the results with no data leaving device. We hope that this will continue the use of 3DGS in the future to be part of the solution to the data problem of generalist robotic policy development in the future.</p></div></section><section class="chapter" id="policy-training"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">5</div><div><h2 class="chapter__title">SceneAgent pipeline: Policy training</h2><p class="chapter__lead">The policy training part of the SceneAgent pipeline starts from a reconstructed intractable simulation environment, which is a Gaussian-splat scene with segmented object meshes and predicted physics, of a robot's workspace. It then converts the simulation environment into manipulation policies that can be deployed on the real robot.</p></div></div><div class="chapter__text"><p>The SceneAgent pipeline generally consists of two major blocks: (i) a demonstration factory that generates thousands of expert episodes with photo-realistic observations inside the twin; (ii) policy training on these episodes alone, and zero-shot transfer to real world environment.</p><p>The demonstration factory creates demonstrations for policy training with two scalable blocks: initial state randomization; and a physics-bounded scripted expert for generating motion control. Randomization creates possible distributions where the reality may be in at large scales: the objects are placed anywhere in the region on the table that the real task uses, the manipulated object's size varies, and the camera and robot base are perturbed around their calibrated poses. Sampled layouts are also physics-bounded: objects are dropped and settled under physics, so every initial state is one that the real table could actually present.</p><p>The scripted expert supplies the actions: it plans from prescient ground-truth object poses inside the simulator and pairs a pick-and-place routine — locate, approach, descend, grasp, lift, carry, lower, release, retreat — to the sampled poses. Finally, a success gate checks whether the object truly ended up in the container and discards episodes that fail. In each recorded episode, observations are rendered by depth-compositing a mesh foreground (robot and objects) over the Gaussian-splat background of the deployment environment.</p><p>The model-agnostic training uses the simulation-generated demonstration episodes for imitation-learning: camera streams are paired with the expert's actions in a standard format, and each episode carries a natural-language instruction sampled from many phrasings of the task. The pipeline then finetunes a pretrained vision language action model on the demonstration dataset via LoRA.</p></div></section><section class="chapter" id="sim2real"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">6</div><div><h2 class="chapter__title">SceneAgent Evaluation in Progress</h2><p class="chapter__lead">After fine-tuning a Vision-Language-Action (VLA) model policy in simulation only, we run the policy in the real world and see improvement in performance. We are still completing the evaluation of our pipeline and will update this with our results soon.</p></div></div><div class="chapter__exhibit"><figure style="margin:0;min-width:0"><div style="position:relative;border:0;background:var(--paper);aspect-ratio:16 / 9;overflow:hidden"><video src="./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.mp4" poster="./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.jpg" muted="" loop="" playsInline="" preload="metadata" aria-label="Evaluation mosaic · episodes 1, 5, and 18 from left to right; real world, simulation, and wrist camera from top to bottom; real-world rollouts sped up to 15 seconds" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;object-position:50% 50%;display:block"></video></div></figure><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--ink)">Policy evaluation in realworld on the Franka Emika Panda after finetuning in simulation.</div><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">6.2</div></div></div><div class="chapter__exhibit sub"><div class="clips" style="grid-template-columns:repeat(3, minmax(0, 1fr))"><figure style="margin:0;min-width:0;font-family:var(--font-sans)"><div style="position:relative;border:1px solid var(--hairline);background:var(--paper);aspect-ratio:842 / 369;overflow:hidden"><img src="./figures/evaluation_layout_1.webp" alt="Trial layout for episode 1: the egg and the basket in plan view about P0" loading="lazy" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;display:block;padding:0"/></div></figure><figure style="margin:0;min-width:0;font-family:var(--font-sans)"><div style="position:relative;border:1px solid var(--hairline);background:var(--paper);aspect-ratio:842 / 369;overflow:hidden"><img src="./figures/evaluation_layout_5.webp" alt="Trial layout for episode 5: the egg and the basket in plan view about P0" loading="lazy" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;display:block;padding:0"/></div></figure><figure style="margin:0;min-width:0;font-family:var(--font-sans)"><div style="position:relative;border:1px solid var(--hairline);background:var(--paper);aspect-ratio:842 / 369;overflow:hidden"><img src="./figures/evaluation_layout_18.webp" alt="Trial layout for episode 18: the egg and the basket in plan view about P0" loading="lazy" style="position:absolute;inset:0;width:100%;height:100%;object-fit:cover;display:block;padding:0"/></div></figure></div><div class="chapter__caption"><div style="font-family:var(--font-sans);font-size:10px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--accent);margin-left:auto;flex-shrink:0">6.3</div></div></div><div class="chapter__text"><p>After finetuning generalist VLAs such as Physical Intelligence’s pi0.5 and NVIDIA’s GR00T 1.6 in simulation only on the digital twin and sisters, we see similar performance in the real world as in simulation. Initial results show a positive increase in task completion rates in the real world after training for 10,000 episodes in simulation of our lab space and example objects processed through our SceneAgent pipeline.</p><p>We are still evaluating the results and are optimistic about the results of real world performance improvements after training in converted 3DGS scenes to use in simulation with predictive physics. We will update our work with the full results soon.</p></div></section><section class="chapter" id="citation"><div class="chapter__rule" aria-hidden="true"></div><div class="chapter__head"><div class="chapter__num" aria-hidden="true">7</div><div><h2 class="chapter__title">Citation</h2></div></div><div class="chapter__exhibit"><div style="font-family:var(--font-sans);background:var(--surface-card);border:1px solid var(--hairline);padding:0;box-shadow:none;color:var(--text-body)"><pre style="font-size:12px;line-height:1.6;padding:20px;color:var(--text-body)">@misc{sceneagent2026,
title = {SceneAgent: 3D Capture-Derived Scenes with Predictive
Physics for Policy Evaluation and Training Environments},
author = {Luke Hollis and Tianxing Fan and Heng Yang},
year = {2026},
note = {Preprint. Computational Robotics Group, Harvard University}
}</pre></div></div><div class="chapter__text"><p>The code, processed scenes, and example converted objects will be released with our paper soon.</p></div><div class="chapter__exhibit sub"><h3 class="chapter__subtitle">Featured Art and 3D Captures</h3><ol class="credits"><li><b>[<!-- -->1<!-- -->]</b><span>3dhdscan, <!-- -->“<a href="https://sketchfab.com/3d-models/rudbeckia-flower-eee1a1ebdb614f38a722b52e26ebe038" target="_blank" rel="noopener noreferrer">Rudbeckia flower</a>,” <!-- -->Sketchfab<!-- -->, CC BY 4.0<!-- -->.</span></li><li><b>[<!-- -->2<!-- -->]</b><span>“<a href="https://poly.cam/capture/71974300-d679-4e71-a7a9-af78c27b2f98" target="_blank" rel="noopener noreferrer">Vintage Sony radio CFS-43</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->3<!-- -->]</b><span>TechnologicalFrostfang2470, <!-- -->“<a href="https://poly.cam/capture/cae4a43a-87aa-4ed8-9e21-cff222662d0b" target="_blank" rel="noopener noreferrer">Totoro</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->4<!-- -->]</b><span>netgis, <!-- -->“<a href="https://sketchfab.com/3d-models/chinese-statue-e0ccdb444522470f9e6567870e63c40c" target="_blank" rel="noopener noreferrer">Chinese statue</a>,” <!-- -->Sketchfab<!-- -->, CC BY-NC 4.0<!-- -->.</span></li><li><b>[<!-- -->5<!-- -->]</b><span>Lassi Kaukonen, <!-- -->“<a href="https://sketchfab.com/3d-models/asus-z170-p-motherboard-b998596cfc4945a0bc7b016005c39321" target="_blank" rel="noopener noreferrer">Asus Z170-P motherboard</a>,” <!-- -->Sketchfab<!-- -->, CC BY 4.0<!-- -->.</span></li><li><b>[<!-- -->6<!-- -->]</b><span>Igor.Jop, <!-- -->“<a href="https://sketchfab.com/3d-models/rx-480-gpu-61e69f50bcd44c7284bef31a8f21c6a7" target="_blank" rel="noopener noreferrer">RX 480 GPU</a>,” <!-- -->Sketchfab<!-- -->, CC BY 4.0<!-- -->.</span></li><li><b>[<!-- -->7<!-- -->]</b><span>“<a href="https://poly.cam/capture/df2dae99-a7ac-4d18-a308-3b7166aa4040" target="_blank" rel="noopener noreferrer">Stegosaurus composite skeleton, American Museum of Natural History</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->8<!-- -->]</b><span>“<a href="https://poly.cam/capture/4e1f9431-425f-49be-844d-bc97a664f2e4" target="_blank" rel="noopener noreferrer">Bencini Comet Universal 444 camera (1977)</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->9<!-- -->]</b><span>wlanfr3ak, <!-- -->“<a href="https://poly.cam/capture/5DF483F7-5ECB-48EA-BDFC-EA4FD93A5A2C" target="_blank" rel="noopener noreferrer">Cessna C172R cockpit</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->10<!-- -->]</b><span>“<a href="https://poly.cam/capture/e762d944-86f8-405e-8cf7-a7872c4097ba" target="_blank" rel="noopener noreferrer">Motorcycle</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->11<!-- -->]</b><span>“<a href="https://poly.cam/capture/0c8cabee-75cf-454e-864a-8c27ef91f41f" target="_blank" rel="noopener noreferrer">Towel rack</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->12<!-- -->]</b><span>“<a href="https://poly.cam/capture/c78e1538-2c3f-4486-ad01-f8eb97098a8a" target="_blank" rel="noopener noreferrer">Red towel</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->13<!-- -->]</b><span>“<a href="https://poly.cam/capture/33170b4a-c8d2-43ef-82d6-9d5c442c7c6a" target="_blank" rel="noopener noreferrer">Dept Artisté anorak</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->14<!-- -->]</b><span>“<a href="https://poly.cam/capture/099a56df-bdb6-429f-89cf-91f1a9bcbee8" target="_blank" rel="noopener noreferrer">Firefighter hose</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->15<!-- -->]</b><span>“<a href="https://poly.cam/capture/72007c87-fa9d-41db-bff6-05e230f68cf7" target="_blank" rel="noopener noreferrer">Electric extension lead</a>,” <!-- -->Polycam<!-- -->.</span></li><li><b>[<!-- -->16<!-- -->]</b><span>“<a href="https://poly.cam/capture/ad4a76f3-2078-48df-81e2-1460cadae520" target="_blank" rel="noopener noreferrer">Liquid Death hat</a>,” <!-- -->Polycam<!-- -->.</span></li></ol></div></section><footer class="colophon"><div class="colophon__field"><div class="colophon__content"><a class="lab-logo" href="https://computationalrobotics.seas.harvard.edu/"><img src="./branding/computational-robotics.jpg" alt="Harvard Computational Robotics Group" width="2286" height="299"/></a><div class="colophon__credit"><div style="font-family:var(--font-sans);font-size:13px;font-weight:700;line-height:1.5;letter-spacing:0.02em;color:var(--gray-cool);text-align:right">Harvard Computational Robotics Group<br/>School of Engineering and Applied Sciences<br/>Harvard University</div></div></div></div></footer></main></div><!--$--><!--/$--><script type="application/ld+json">{"@context":"https://schema.org","@type":"ScholarlyArticle","headline":"SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments","name":"SceneAgent","description":"SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training.","abstract":"SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS.","author":[{"@type":"Person","name":"Luke Hollis","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}},{"@type":"Person","name":"Tianxing Fan","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}},{"@type":"Person","name":"Heng Yang","affiliation":{"@type":"Organization","name":"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University","url":"https://computationalrobotics.seas.harvard.edu/"}}],"publisher":{"@type":"Organization","name":"Harvard Computational Robotics Group","url":"https://computationalrobotics.seas.harvard.edu/"},"datePublished":"2026","creativeWorkStatus":"Preprint","image":"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg","url":"https://computationalrobotics.seas.harvard.edu/SceneAgent/","mainEntityOfPage":"https://computationalrobotics.seas.harvard.edu/SceneAgent/","inLanguage":"en","keywords":"SceneAgent, real-to-sim, sim-to-real, 3D Gaussian Splatting, simulation environments, predictive physics, robot policy evaluation, policy training, online planning, object segmentation, articulation, deformable objects, digital sisters, robot learning, Harvard Computational Robotics Group"}</script><script src="./_next/static/chunks/0ba1ebj8wcm3s.js" id="_R_" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:\"$Sreact.fragment\"\n2:I[28422,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"default\"]\n3:I[4703,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"default\"]\n7:I[76400,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"default\",1]\n:HL[\"./_next/static/chunks/1zf8qs8h9vuqk.css\",\"style\"]\n:HL[\"./_next/static/media/PlexMono_Regular-s.p.2x6m9-apme3b_.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n:HL[\"./_next/static/media/Silkscreen_Regular-s.p.3es8zrs5x_4vu.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n4:Tad0,"])</script><script>self.__next_f.push([1,"{\"@context\":\"https://schema.org\",\"@type\":\"ScholarlyArticle\",\"headline\":\"SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments\",\"name\":\"SceneAgent\",\"description\":\"SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training.\",\"abstract\":\"SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS.\",\"author\":[{\"@type\":\"Person\",\"name\":\"Luke Hollis\",\"affiliation\":{\"@type\":\"Organization\",\"name\":\"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University\",\"url\":\"https://computationalrobotics.seas.harvard.edu/\"}},{\"@type\":\"Person\",\"name\":\"Tianxing Fan\",\"affiliation\":{\"@type\":\"Organization\",\"name\":\"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University\",\"url\":\"https://computationalrobotics.seas.harvard.edu/\"}},{\"@type\":\"Person\",\"name\":\"Heng Yang\",\"affiliation\":{\"@type\":\"Organization\",\"name\":\"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University\",\"url\":\"https://computationalrobotics.seas.harvard.edu/\"}}],\"publisher\":{\"@type\":\"Organization\",\"name\":\"Harvard Computational Robotics Group\",\"url\":\"https://computationalrobotics.seas.harvard.edu/\"},\"datePublished\":\"2026\",\"creativeWorkStatus\":\"Preprint\",\"image\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg\",\"url\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/\",\"mainEntityOfPage\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/\",\"inLanguage\":\"en\",\"keywords\":\"SceneAgent, real-to-sim, sim-to-real, 3D Gaussian Splatting, simulation environments, predictive physics, robot policy evaluation, policy training, online planning, object segmentation, articulation, deformable objects, digital sisters, robot learning, Harvard Computational Robotics Group\"}"])</script><script>self.__next_f.push([1,"0:{\"P\":null,\"c\":[\"\",\"\"],\"q\":\"\",\"i\":false,\"f\":[[[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",16],[[\"$\",\"$1\",\"c\",{\"children\":[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"./_next/static/chunks/1zf8qs8h9vuqk.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}],[\"$\",\"script\",\"script-0\",{\"src\":\"./_next/static/chunks/3d-g45tvvtwds.js\",\"async\":true,\"nonce\":\"$undefined\"}]],[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"silkscreen_5c214b00-module__BDnqEG__variable plexmono_301f1454-module__3YJGwG__variable\",\"children\":[\"$\",\"body\",null,{\"children\":[[\"$\",\"$L2\",null,{\"parallelRouterKey\":\"children\",\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L3\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":404}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],[]],\"forbidden\":\"$undefined\",\"unauthorized\":\"$undefined\"}],[\"$\",\"script\",null,{\"type\":\"application/ld+json\",\"dangerouslySetInnerHTML\":{\"__html\":\"$4\"}}]]}]}]]}],{\"children\":[\"$L5\",{},null,false,null]},null,false,null],\"$L6\",false]],\"m\":\"$undefined\",\"G\":[\"$7\",[\"$L8\"]],\"S\":true,\"h\":null,\"s\":\"$undefined\",\"l\":\"$undefined\",\"p\":\"$undefined\",\"d\":\"$undefined\",\"b\":\"F18sWxiUtr6B58sAiA9h4\"}\n"])</script><script>self.__next_f.push([1,"9:I[88409,[\"./_next/static/chunks/3d-g45tvvtwds.js\",\"./_next/static/chunks/24ble59yb_9_o.js\",\"./_next/static/chunks/36hy0ec4-ck76.js\",\"./_next/static/chunks/0p_6srangbw9h.js\",\"./_next/static/chunks/0_sx6di6mxk5o.js\"],\"SideNav\"]\na:I[90112,[\"./_next/static/chunks/3d-g45tvvtwds.js\",\"./_next/static/chunks/24ble59yb_9_o.js\",\"./_next/static/chunks/36hy0ec4-ck76.js\",\"./_next/static/chunks/0p_6srangbw9h.js\",\"./_next/static/chunks/0_sx6di6mxk5o.js\"],\"VideoPlate\"]\nb:I[45828,[\"./_next/static/chunks/3d-g45tvvtwds.js\",\"./_next/static/chunks/24ble59yb_9_o.js\",\"./_next/static/chunks/36hy0ec4-ck76.js\",\"./_next/static/chunks/0p_6srangbw9h.js\",\"./_next/static/chunks/0_sx6di6mxk5o.js\"],\"Button\"]\nc:I[7838,[\"./_next/static/chunks/3d-g45tvvtwds.js\",\"./_next/static/chunks/24ble59yb_9_o.js\",\"./_next/static/chunks/36hy0ec4-ck76.js\",\"./_next/static/chunks/0p_6srangbw9h.js\",\"./_next/static/chunks/0_sx6di6mxk5o.js\"],\"PipelineChapter\"]\n1d:I[36215,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"ViewportBoundary\"]\n1f:I[36215,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"MetadataBoundary\"]\n20:\"$Sreact.suspense\"\n:HL[\"./branding/computational-robotics.jpg\",\"image\"]\n"])</script><script>self.__next_f.push([1,"5:[\"$\",\"$1\",\"c\",{\"children\":[[\"$\",\"div\",null,{\"className\":\"shell\",\"children\":[[\"$\",\"$L9\",null,{}],[\"$\",\"main\",null,{\"children\":[[\"$\",\"section\",null,{\"id\":\"top\",\"style\":{\"background\":\"var(--paper)\",\"padding\":\"40px var(--chapter-pad) 56px\"},\"children\":[\"$\",\"div\",null,{\"style\":{\"maxWidth\":\"calc(var(--page-w) - 2 * var(--chapter-pad))\"},\"children\":[[\"$\",\"div\",null,{\"className\":\"cover__brand\",\"children\":[\"$\",\"a\",null,{\"className\":\"lab-logo\",\"href\":\"https://computationalrobotics.seas.harvard.edu/\",\"children\":[\"$\",\"img\",null,{\"src\":\"./branding/computational-robotics.jpg\",\"alt\":\"Harvard Computational Robotics Group\",\"width\":2286,\"height\":299}]}]}],[\"$\",\"div\",null,{\"style\":{\"borderTop\":\"3px solid var(--ink)\",\"paddingTop\":10,\"display\":\"flex\",\"justifyContent\":\"space-between\",\"fontSize\":13,\"lineHeight\":1.45,\"color\":\"var(--ink)\"},\"children\":[[\"$\",\"div\",null,{\"children\":[\"Harvard Computational Robotics Group\",[\"$\",\"br\",null,{}],\"School of Engineering and Applied Sciences, Harvard University\"]}],[\"$\",\"div\",null,{\"style\":{\"textAlign\":\"right\"},\"children\":[\"Preprint\",[\"$\",\"br\",null,{}],\"2026\"]}]]}],[\"$\",\"$La\",null,{\"src\":\"./videos/cover_video.mp4\",\"poster\":\"./videos/cover_video.jpg\",\"ratio\":\"16 / 9\",\"alt\":\"SceneAgent cover video\",\"controls\":false,\"style\":{\"marginTop\":40}}],[\"$\",\"h1\",null,{\"className\":\"cover__title\",\"children\":\"SceneAgent\"}],[\"$\",\"div\",null,{\"style\":{\"marginTop\":12,\"maxWidth\":640,\"fontSize\":14,\"color\":\"var(--text-muted)\"},\"children\":\"An agentic pipeline for converting 3D captures to simulation environments with predictive physics for policy evaluation, online planning, and policy training\"}],[\"$\",\"div\",null,{\"style\":{\"marginTop\":24,\"fontSize\":15,\"fontWeight\":700,\"color\":\"var(--ink)\"},\"children\":[[\"$\",\"span\",\"Luke Hollis\",{\"children\":[[\"$\",\"a\",null,{\"href\":\"mailto:lhollis@g.harvard.edu\",\"children\":\"Luke Hollis\"}],[\"$\",\"sup\",null,{\"children\":[1,\"*\"]}],\", \"]}],[\"$\",\"span\",\"Tianxing Fan\",{\"children\":[[\"$\",\"a\",null,{\"href\":\"mailto:tianxing_fan@seas.harvard.edu\",\"children\":\"Tianxing Fan\"}],[\"$\",\"sup\",null,{\"children\":[1,\"*\"]}],\", \"]}],[\"$\",\"span\",\"Heng Yang\",{\"children\":[[\"$\",\"a\",null,{\"href\":\"mailto:hankyang@seas.harvard.edu\",\"children\":\"Heng Yang\"}],[\"$\",\"sup\",null,{\"children\":[1,\"\"]}],\"\"]}]]}],[\"$\",\"div\",null,{\"style\":{\"marginTop\":8,\"fontSize\":13,\"lineHeight\":1.5,\"color\":\"var(--text-muted)\"},\"children\":[[[\"$\",\"span\",\"1\",{\"children\":[[\"$\",\"sup\",null,{\"children\":1}],\" \",[\"$\",\"a\",null,{\"href\":\"https://computationalrobotics.seas.harvard.edu/\",\"target\":\"_blank\",\"rel\":\"noopener\",\"children\":\"Harvard Computational Robotics Group, School of Engineering and Applied Sciences, Harvard University\"}]]}]],[\"$\",\"br\",null,{}],[\"$\",\"sup\",null,{\"children\":\"*\"}],\" \",\"Equal contribution\"]}],[\"$\",\"div\",null,{\"style\":{\"marginTop\":24,\"display\":\"flex\",\"alignItems\":\"center\",\"gap\":16,\"flexWrap\":\"wrap\"},\"children\":[[\"$\",\"$Lb\",null,{\"size\":\"sm\",\"disabled\":true,\"style\":{\"textTransform\":\"uppercase\"},\"children\":\"Paper\"}],[\"$\",\"$Lb\",null,{\"size\":\"sm\",\"variant\":\"secondary\",\"disabled\":true,\"style\":{\"textTransform\":\"uppercase\"},\"children\":\"Code\"}],[\"$\",\"span\",null,{\"style\":{\"fontSize\":12,\"textTransform\":\"uppercase\",\"color\":\"var(--text-muted)\"},\"children\":\"[coming soon]\"}]]}],[\"$\",\"p\",null,{\"className\":\"chapter__text\",\"children\":\"SceneAgent converts 3d captures of real-world scenes into simulation environments that can be used for policy evaluation, online planning, and policy training or finetuning. The pipeline primarily focuses on converting environments captured with 3d Gaussian Splatting (3DGS) but also supports scenes captured with photogrammetry and LiDAR. It incorporates 3d semantic features, object segmentation, predictive per-gaussian physics properties, object decomposition, deformability, and articulation. To increase policy generalization, it is also able to generate “digital sisters” or similar versions of each object within an environment. We are still evaluating how well policies trained in this environment perform in the real world, and initial results show promising success rates similar to or improving on other recent papers such as SimFoundry and PolaRiS.\"}]]}]}],[\"$\",\"$Lc\",null,{}],[\"$\",\"section\",null,{\"className\":\"chapter\",\"id\":\"cousins\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"2\"}],\"$Ld\"]}],[\"$Le\",\"$Lf\",\"$L10\",\"$L11\"]]}],\"$L12\",\"$L13\",\"$L14\",\"$L15\",\"$L16\",\"$L17\"]}]]}],[\"$L18\",\"$L19\",\"$L1a\",\"$L1b\"],\"$L1c\"]}]\n"])</script><script>self.__next_f.push([1,"6:[\"$\",\"$1\",\"h\",{\"children\":[null,[\"$\",\"$L1d\",null,{\"children\":\"$L1e\"}],[\"$\",\"div\",null,{\"hidden\":true,\"children\":[\"$\",\"$L1f\",null,{\"children\":[\"$\",\"$20\",null,{\"name\":\"Next.Metadata\",\"children\":\"$L21\"}]}]}],[\"$\",\"meta\",null,{\"name\":\"next-size-adjust\",\"content\":\"\"}]]}]\n8:[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"./_next/static/chunks/1zf8qs8h9vuqk.css\",\"precedence\":\"next\",\"crossOrigin\":\"$undefined\",\"nonce\":\"$undefined\"}]\n"])</script><script>self.__next_f.push([1,"22:I[79999,[\"./_next/static/chunks/3d-g45tvvtwds.js\",\"./_next/static/chunks/24ble59yb_9_o.js\",\"./_next/static/chunks/36hy0ec4-ck76.js\",\"./_next/static/chunks/0p_6srangbw9h.js\",\"./_next/static/chunks/0_sx6di6mxk5o.js\"],\"CousinsGallery\"]\n44:I[36215,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"OutletBoundary\"]\nd:[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"Digital Twins and Sisters\"}],[\"$\",\"p\",null,{\"className\":\"chapter__lead\",\"children\":\"In order to increase training generalizability from simulation back to the real world, we create slight variations of each object in the scenes converted by the pipeline.\"}]]}]\ne:[\"$\",\"div\",null,{\"className\":\"chapter__exhibit\",\"data-bleed\":\"$undefined\",\"children\":[[\"$\",\"$L22\",null,{}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"2.2\"}]]}]]}]\nf:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"Instead of digital twins, these are rather “digital sisters,” not exact replicas of reality but similar variations that reflect the range of diversity that a manipulator may encounter in real-world scenarios. We also randomize placement of objects and lighting to continue to improve policy performance in real-world rollouts.\"}],[\"$\",\"p\",null,{\"children\":\"In our initial testing, this increases real-world task completion rates after training in our converted simulation digital sister scenes, but we are continuing to test and evaluate how different lighting, placement, and generative 3d models influence policy performance from simulation back to the real world.\"}],[\"$\",\"p\",null,{\"children\":\"We chose to randomize each of these factors when composing full digital sister scenes, but future work could test each in isolation to verify its influence on policy generalizability during finetuning to study which is best to spend more iterations on.\"}]]}]\n10:[\"$\",\"div\",null,{\"className\":\"chapter__exhibit sub\",\"data-bleed\":\"$undefined\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(2, minmax(0, 1fr))\"},\"children\":[[\"$\",\"$La\",\"warehouse_suitcase_suitcase\",{\"src\":\"./videos/warehouse_suitcase_suitcase.mp4\",\"poster\":\"./videos/warehouse_suitcase_suitcase.jpg\",\"caption\":\"Contact rich tasks with predictive physics, such as rolling a red rubber ball from one side of the suitcase to the other\",\"ratio\":\"4 / 3\",\"alt\":\"Warehouse · a different suitcase\"}],[\"$\",\"$La\",\"studio_flower_noir_cinema\",{\"src\":\"./videos/studio_flower_noir_cinema.mp4\",\"poster\":\"./videos/studio_flower_noir_cinema.jpg\",\"caption\":\"Picking up delicate items such as the flower 3d capture from the previous greenhouse scene in hybrid digital art and 3d capture environments\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · flower grasp, relit\"}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"2.3\"}]]}]]}]\n11:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"Along with the typical tasks being trained in simulation policy, we are seeking to understand better how the predictive physics properties can help us achieve a series of contact-rich tasks, such as rolling a red rubber ball from one side of the suitcase to another with a gripper on the Franka arm or picking a columnar object from a tightly packed box.\"}],[\"$\",\"p\",null,{\"children\":\"In future iterations, we also hope to study further how small perturbations of the physics properties may also influence training in simulation. For example, if an object’s mass, friction coefficient, or rigidity are varied slightly between versions, will this help our policy during finetuning more accurately grasp and manipulate deformable ob"])</script><script>self.__next_f.push([1,"jects or objects with heterogeneous physics properties (such as shoes, hats, cardboard boxes, and similar).\"}]]}]\n"])</script><script>self.__next_f.push([1,"12:[\"$\",\"section\",null,{\"className\":\"chapter chapter--bleed\",\"id\":\"hybrid\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"3\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"Hybrid Scenes with 3d Captures and Digital Art\"}],[\"$\",\"p\",null,{\"className\":\"chapter__lead\",\"children\":\"We convert individual 3d captured objects into simulation assets with our SceneAgent pipeline and incorporate them into synthetic scenes prepared by 3d artists.\"}]]}]]}],[[[\"$\",\"$1\",\"scenes\",{\"children\":[false,[\"$\",\"div\",null,{\"className\":\"chapter__exhibit chapter__exhibit--bleed\",\"data-bleed\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(3, minmax(0, 1fr))\"},\"children\":[[\"$\",\"$La\",\"kitchen_apple\",{\"src\":\"./videos/kitchen_apple.mp4\",\"poster\":\"./videos/kitchen_apple.jpg\",\"caption\":\"Hybrid scenes from available assets in Isaac Lab, a Kitchen scene picking fruit, similar to RoboCasa\",\"ratio\":\"4 / 3\",\"alt\":\"Kitchen · apple\"}],[\"$\",\"$La\",\"warehouse_pack\",{\"src\":\"./videos/warehouse_pack.mp4\",\"poster\":\"./videos/warehouse_pack.jpg\",\"caption\":\"Contact rich tasks picking tightly packed vertical cylinders\",\"ratio\":\"4 / 3\",\"alt\":\"Warehouse · pack\"}],[\"$\",\"$La\",\"chem_lab_sort\",{\"src\":\"./videos/chem_lab_sort.mp4\",\"poster\":\"./videos/chem_lab_sort.jpg\",\"caption\":\"Chemistry lab, sorting vials into beakers\",\"ratio\":\"4 / 3\",\"alt\":\"Chemistry lab · sort\"}],[\"$\",\"$La\",\"datacenter_cable\",{\"src\":\"./videos/datacenter_cable.mp4\",\"poster\":\"./videos/datacenter_cable.jpg\",\"caption\":\"Picking and plugging cables into a data center server rack\",\"ratio\":\"4 / 3\",\"alt\":\"Data center · cable\"}],[\"$\",\"$La\",\"studio_cardboard_fold\",{\"src\":\"./videos/studio_cardboard_fold.mp4\",\"poster\":\"./videos/studio_cardboard_fold.jpg\",\"caption\":\"Two arms folding a cardboard packaging in simulation, packing a deformable object\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · cardboard fold\"}],[\"$\",\"$La\",\"studio_succulent_watering\",{\"src\":\"./videos/studio_succulent_watering.mp4\",\"poster\":\"./videos/studio_succulent_watering.jpg\",\"caption\":\"Hybrid digital art and 3d capture scene with a Franka Emika Panda arm watering a succulent with an eyedropper\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · succulent watering\"}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"3.2\"}]]}]]}]]}],[\"$\",\"$1\",\"hands\",{\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"Because many backgrounds and environments are redundant across common tasks, we combine specific 3d captured objects processed with our SceneAgent pipeline with other existing art assets created by 3d artists, such as in the NVIDIA Omniverse asset repository. This allows for rapid development of common scenes for policy evaluation along with scenes with specific objects for individual tasks.\"}],[\"$\",\"p\",null,{\"children\":\"For example, when a manipulator may need to learn how to use a specific part, say for a new healthcare device or a new cooling fan in a server rack, that specific part individually could be 3d captured and processed in our pipeline and then incorporated into a pre-existing hospital or data center scene such as the default environments that accompany NVIDIA Isaac Lab.\"}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__exhibit chapter__exhibit--bleed sub\",\"data-bleed\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(3, minmax(0, 1fr))\"},\"children\":[[\"$\",\"$La\",\"sharpa_hand_flower_scoop_realtime\",{\"src\":\"./videos/sharpa_hand_flower_scoop_realtime.mp4\",\"poster\":\"./videos/sharpa_hand_flower_scoop_realtime.jpg\",\"caption\":\"A Sharpa dexterous hand scooping to cup a delicate flower in its palm\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · Sharpa hand, real time\"}],\"$L23\",\"$L24\",\"$L25\",\"$L26\",\"$L27\"]}],\"$L28\"]}]]}]],\"$L29\"]]}]\n"])</script><script>self.__next_f.push([1,"13:[\"$\",\"section\",null,{\"className\":\"chapter chapter--bleed\",\"id\":\"agents\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"4\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"Agentic Pipeline for SceneAgent\"}],[\"$\",\"p\",null,{\"className\":\"chapter__lead\",\"children\":\"We provide a team of agents to convert many types of 3d captures into fully usable simulation scenes ready for training in Isaac Lab, MuJoCo, or Unreal. The easiest way to convert a 3d capture is by an agent swarm that processes each step of the pipeline and visually inspects and reviews it for errors.\"}]]}]]}],[[\"$\",\"div\",null,{\"className\":\"chapter__exhibit chapter__exhibit--bleed\",\"data-bleed\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"spread\",\"role\":\"img\",\"aria-label\":\"The agentic workflow: nine agents in three stages, Input, Conversion and Sisters, processing 3d capture data into usable simulation scenes.\",\"children\":[\"$\",\"div\",null,{\"className\":\"spread__grid\",\"style\":{\"--spread-fluid\":\"minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr) minmax(0, 144fr) minmax(0, 480fr)\",\"--spread-fixed\":\"calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s)) minmax(0, 1fr) calc(480px * var(--s))\"},\"children\":[[\"$\",\"$1\",\"00\",{\"children\":[[\"$\",\"svg\",null,{\"className\":\"spread__col\",\"viewBox\":\"0 0 480 932\",\"style\":{\"fontFamily\":\"Helvetica, \\\"Helvetica Neue\\\", Arimo, Arial, sans-serif\"},\"children\":[[\"$\",\"g\",null,{\"children\":[[\"$\",\"text\",null,{\"x\":0,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"00\"}],[\"$\",\"text\",null,{\"x\":46,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Input\"}],[\"$\",\"line\",null,{\"x1\":0,\"y1\":38.5,\"x2\":480,\"y2\":38.5,\"stroke\":\"var(--ink)\"}],[[\"$\",\"text\",\"0\",{\"x\":0,\"y\":64,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"Process input 3d data from a diverse\"}],[\"$\",\"text\",\"1\",{\"x\":0,\"y\":92,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"range of sources, including RGB images,\"}],[\"$\",\"text\",\"2\",{\"x\":0,\"y\":120,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"pre-processed photogrammetry files,\"}],[\"$\",\"text\",\"3\",{\"x\":0,\"y\":148,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"or LiDAR\"}]]]}],[[\"$\",\"g\",\"01\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"01 Orchestrator: Plans the full conversion pipeline based on the status of the input data\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":168.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 184) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":0},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-6\",{\"x\":6,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"1-6\",{\"x\":6,\"y\":1,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"2-6\",{\"x\":6,\"y\":2,\"width\":1,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"01\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Orchestrator\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":265,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Plans the full conversion pipeline based\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":291,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"on the status of the input data\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":320,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-RUN-PIPELINE\"}]]}],[\"$\",\"line\",null,{\"x1\":240,\"y1\":334,\"x2\":240,\"y2\":362,\"stroke\":\"var(--text-label)\"}]]}],[\"$\",\"g\",\"02\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"02 Ingest: Process input RGB or other data sources to 3DGS\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":362.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 378) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":\"$L2a\"}],\"$L2b\",\"$L2c\",[\"$L2d\",\"$L2e\"],\"$L2f\"]}],\"$L30\"]}],\"$L31\"]]}],\"$L32\"]}],\"$L33\",\"$L34\"]}]}],\"$L35\"]}],\"$L36\",\"$L37\",\"$L38\",\"$L39\",\"$L3a\",\"$L3b\"]]}]\n"])</script><script>self.__next_f.push([1,"14:[\"$\",\"section\",null,{\"className\":\"chapter\",\"id\":\"policy-training\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"5\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"SceneAgent pipeline: Policy training\"}],[\"$\",\"p\",null,{\"className\":\"chapter__lead\",\"children\":\"The policy training part of the SceneAgent pipeline starts from a reconstructed intractable simulation environment, which is a Gaussian-splat scene with segmented object meshes and predicted physics, of a robot's workspace. It then converts the simulation environment into manipulation policies that can be deployed on the real robot.\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"The SceneAgent pipeline generally consists of two major blocks: (i) a demonstration factory that generates thousands of expert episodes with photo-realistic observations inside the twin; (ii) policy training on these episodes alone, and zero-shot transfer to real world environment.\"}],[\"$\",\"p\",null,{\"children\":\"The demonstration factory creates demonstrations for policy training with two scalable blocks: initial state randomization; and a physics-bounded scripted expert for generating motion control. Randomization creates possible distributions where the reality may be in at large scales: the objects are placed anywhere in the region on the table that the real task uses, the manipulated object's size varies, and the camera and robot base are perturbed around their calibrated poses. Sampled layouts are also physics-bounded: objects are dropped and settled under physics, so every initial state is one that the real table could actually present.\"}],[\"$\",\"p\",null,{\"children\":\"The scripted expert supplies the actions: it plans from prescient ground-truth object poses inside the simulator and pairs a pick-and-place routine — locate, approach, descend, grasp, lift, carry, lower, release, retreat — to the sampled poses. Finally, a success gate checks whether the object truly ended up in the container and discards episodes that fail. In each recorded episode, observations are rendered by depth-compositing a mesh foreground (robot and objects) over the Gaussian-splat background of the deployment environment.\"}],[\"$\",\"p\",null,{\"children\":\"The model-agnostic training uses the simulation-generated demonstration episodes for imitation-learning: camera streams are paired with the expert's actions in a standard format, and each episode carries a natural-language instruction sampled from many phrasings of the task. The pipeline then finetunes a pretrained vision language action model on the demonstration dataset via LoRA.\"}]]}]]}]\n"])</script><script>self.__next_f.push([1,"15:[\"$\",\"section\",null,{\"className\":\"chapter\",\"id\":\"sim2real\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"6\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"SceneAgent Evaluation in Progress\"}],[\"$\",\"p\",null,{\"className\":\"chapter__lead\",\"children\":\"After fine-tuning a Vision-Language-Action (VLA) model policy in simulation only, we run the policy in the real world and see improvement in performance. We are still completing the evaluation of our pipeline and will update this with our results soon.\"}]]}]]}],[[\"$\",\"div\",null,{\"className\":\"chapter__exhibit\",\"data-bleed\":\"$undefined\",\"children\":[[\"$\",\"$La\",\"evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s\",{\"src\":\"./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.mp4\",\"poster\":\"./videos/evaluation/real_sim_wrist_episodes_1_5_18_mosaic_real15s.jpg\",\"ratio\":\"16 / 9\",\"alt\":\"Evaluation mosaic · episodes 1, 5, and 18 from left to right; real world, simulation, and wrist camera from top to bottom; real-world rollouts sped up to 15 seconds\",\"style\":{\"border\":0,\"background\":\"var(--paper)\"}}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--ink)\"},\"children\":\"Policy evaluation in realworld on the Franka Emika Panda after finetuning in simulation.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"6.2\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__exhibit sub\",\"data-bleed\":\"$undefined\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(3, minmax(0, 1fr))\"},\"children\":[[\"$\",\"figure\",\"evaluation_layout_1\",{\"style\":{\"margin\":0,\"minWidth\":0,\"fontFamily\":\"var(--font-sans)\"},\"children\":[[\"$\",\"div\",null,{\"style\":{\"position\":\"relative\",\"border\":\"1px solid var(--hairline)\",\"background\":\"var(--paper)\",\"aspectRatio\":\"842 / 369\",\"overflow\":\"hidden\"},\"children\":[[\"$\",\"img\",null,{\"src\":\"./figures/evaluation_layout_1.webp\",\"alt\":\"Trial layout for episode 1: the egg and the basket in plan view about P0\",\"loading\":\"lazy\",\"style\":{\"position\":\"absolute\",\"inset\":0,\"width\":\"100%\",\"height\":\"100%\",\"objectFit\":\"cover\",\"objectPosition\":\"$undefined\",\"display\":\"block\",\"padding\":0}}],false,false,\"$undefined\"]}],\"$undefined\"]}],[\"$\",\"figure\",\"evaluation_layout_5\",{\"style\":{\"margin\":0,\"minWidth\":0,\"fontFamily\":\"var(--font-sans)\"},\"children\":[[\"$\",\"div\",null,{\"style\":{\"position\":\"relative\",\"border\":\"1px solid var(--hairline)\",\"background\":\"var(--paper)\",\"aspectRatio\":\"842 / 369\",\"overflow\":\"hidden\"},\"children\":[[\"$\",\"img\",null,{\"src\":\"./figures/evaluation_layout_5.webp\",\"alt\":\"Trial layout for episode 5: the egg and the basket in plan view about P0\",\"loading\":\"lazy\",\"style\":{\"position\":\"absolute\",\"inset\":0,\"width\":\"100%\",\"height\":\"100%\",\"objectFit\":\"cover\",\"objectPosition\":\"$undefined\",\"display\":\"block\",\"padding\":0}}],false,false,\"$undefined\"]}],\"$undefined\"]}],[\"$\",\"figure\",\"evaluation_layout_18\",{\"style\":{\"margin\":0,\"minWidth\":0,\"fontFamily\":\"var(--font-sans)\"},\"children\":[[\"$\",\"div\",null,{\"style\":{\"position\":\"relative\",\"border\":\"1px solid var(--hairline)\",\"background\":\"var(--paper)\",\"aspectRatio\":\"842 / 369\",\"overflow\":\"hidden\"},\"children\":[[\"$\",\"img\",null,{\"src\":\"./figures/evaluation_layout_18.webp\",\"alt\":\"Trial layout for episode 18: the egg and the basket in plan view about P0\",\"loading\":\"lazy\",\"style\":{\"position\":\"absolute\",\"inset\":0,\"width\":\"100%\",\"height\":\"100%\",\"objectFit\":\"cover\",\"objectPosition\":\"$undefined\",\"display\":\"block\",\"padding\":0}}],false,false,\"$undefined\"]}],\"$undefined\"]}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"6.3\"}]]}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"After finetuning generalist VLAs such as Physical Intelligence’s pi0.5 and NVIDIA’s GR00T 1.6 in simulation only on the digital twin and sisters, we see similar performance in the real world as in simulation. Initial results show a positive increase in task completion rates in the real world after training for 10,000 episodes in simulation of our lab space and example objects processed through our SceneAgent pipeline.\"}],\"$L3c\"]}]]]}]\n"])</script><script>self.__next_f.push([1,"16:[\"$\",\"section\",null,{\"className\":\"chapter\",\"id\":\"citation\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__rule\",\"aria-hidden\":\"true\"}],[\"$\",\"div\",null,{\"className\":\"chapter__head\",\"children\":[[\"$\",\"div\",null,{\"className\":\"chapter__num\",\"aria-hidden\":\"true\",\"children\":\"7\"}],[\"$\",\"div\",null,{\"children\":[[\"$\",\"h2\",null,{\"className\":\"chapter__title\",\"children\":\"Citation\"}],\"$undefined\"]}]]}],[[\"$\",\"div\",null,{\"className\":\"chapter__exhibit\",\"data-bleed\":\"$undefined\",\"children\":[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"background\":\"var(--surface-card)\",\"border\":\"1px solid var(--hairline)\",\"padding\":0,\"boxShadow\":\"none\",\"color\":\"var(--text-body)\"},\"children\":[\"$\",\"pre\",null,{\"style\":{\"fontSize\":12,\"lineHeight\":1.6,\"padding\":20,\"color\":\"var(--text-body)\"},\"children\":\"@misc{sceneagent2026,\\n title = {SceneAgent: 3D Capture-Derived Scenes with Predictive\\n Physics for Policy Evaluation and Training Environments},\\n author = {Luke Hollis and Tianxing Fan and Heng Yang},\\n year = {2026},\\n note = {Preprint. Computational Robotics Group, Harvard University}\\n}\"}]}]}],[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[\"$\",\"p\",null,{\"children\":\"The code, processed scenes, and example converted objects will be released with our paper soon.\"}]}],[\"$\",\"div\",null,{\"className\":\"chapter__exhibit sub\",\"data-bleed\":\"$undefined\",\"children\":[[\"$\",\"h3\",null,{\"className\":\"chapter__subtitle\",\"children\":\"Featured Art and 3D Captures\"}],[\"$\",\"ol\",null,{\"className\":\"credits\",\"children\":[[\"$\",\"li\",\"flower\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",1,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"3dhdscan, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://sketchfab.com/3d-models/rudbeckia-flower-eee1a1ebdb614f38a722b52e26ebe038\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Rudbeckia flower\"}],\",” \",\"Sketchfab\",\", CC BY 4.0\",\".\"]}]]}],[\"$\",\"li\",\"radio\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",2,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/71974300-d679-4e71-a7a9-af78c27b2f98\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Vintage Sony radio CFS-43\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}],[\"$\",\"li\",\"totoro\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",3,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"TechnologicalFrostfang2470, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/cae4a43a-87aa-4ed8-9e21-cff222662d0b\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Totoro\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}],[\"$\",\"li\",\"statue\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",4,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"netgis, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://sketchfab.com/3d-models/chinese-statue-e0ccdb444522470f9e6567870e63c40c\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Chinese statue\"}],\",” \",\"Sketchfab\",\", CC BY-NC 4.0\",\".\"]}]]}],[\"$\",\"li\",\"motherboard\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",5,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"Lassi Kaukonen, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://sketchfab.com/3d-models/asus-z170-p-motherboard-b998596cfc4945a0bc7b016005c39321\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Asus Z170-P motherboard\"}],\",” \",\"Sketchfab\",\", CC BY 4.0\",\".\"]}]]}],[\"$\",\"li\",\"gpu\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",6,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"Igor.Jop, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://sketchfab.com/3d-models/rx-480-gpu-61e69f50bcd44c7284bef31a8f21c6a7\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"RX 480 GPU\"}],\",” \",\"Sketchfab\",\", CC BY 4.0\",\".\"]}]]}],[\"$\",\"li\",\"stegosaurus\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",7,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/df2dae99-a7ac-4d18-a308-3b7166aa4040\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Stegosaurus composite skeleton, American Museum of Natural History\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}],[\"$\",\"li\",\"bencini\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",8,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/4e1f9431-425f-49be-844d-bc97a664f2e4\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Bencini Comet Universal 444 camera (1977)\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}],[\"$\",\"li\",\"cessna\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",9,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"wlanfr3ak, \",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/5DF483F7-5ECB-48EA-BDFC-EA4FD93A5A2C\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Cessna C172R cockpit\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}],\"$L3d\",\"$L3e\",\"$L3f\",\"$L40\",\"$L41\",\"$L42\",\"$L43\"]}]]}]]]}]\n"])</script><script>self.__next_f.push([1,"17:[\"$\",\"footer\",null,{\"className\":\"colophon\",\"children\":[\"$\",\"div\",null,{\"className\":\"colophon__field\",\"children\":[\"$\",\"div\",null,{\"className\":\"colophon__content\",\"children\":[[\"$\",\"a\",null,{\"className\":\"lab-logo\",\"href\":\"https://computationalrobotics.seas.harvard.edu/\",\"children\":[\"$\",\"img\",null,{\"src\":\"./branding/computational-robotics.jpg\",\"alt\":\"Harvard Computational Robotics Group\",\"width\":2286,\"height\":299}]}],[\"$\",\"div\",null,{\"className\":\"colophon__credit\",\"children\":[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":13,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--gray-cool)\",\"textAlign\":\"right\"},\"children\":[\"Harvard Computational Robotics Group\",[\"$\",\"br\",null,{}],\"School of Engineering and Applied Sciences\",[\"$\",\"br\",null,{}],\"Harvard University\"]}]}]]}]}]}]\n18:[\"$\",\"script\",\"script-0\",{\"src\":\"./_next/static/chunks/24ble59yb_9_o.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n19:[\"$\",\"script\",\"script-1\",{\"src\":\"./_next/static/chunks/36hy0ec4-ck76.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n1a:[\"$\",\"script\",\"script-2\",{\"src\":\"./_next/static/chunks/0p_6srangbw9h.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n1b:[\"$\",\"script\",\"script-3\",{\"src\":\"./_next/static/chunks/0_sx6di6mxk5o.js\",\"async\":true,\"nonce\":\"$undefined\"}]\n1c:[\"$\",\"$L44\",null,{\"children\":[\"$\",\"$20\",null,{\"name\":\"Next.MetadataOutlet\",\"children\":\"$@45\"}]}]\n"])</script><script>self.__next_f.push([1,"23:[\"$\",\"$La\",\"hands_clipping_in_gpu\",{\"src\":\"./videos/hands_clipping_in_gpu.mp4\",\"poster\":\"./videos/hands_clipping_in_gpu.jpg\",\"caption\":\"Two dextrous Sharpa hands clipping a GPU into place into a motherboard\",\"ratio\":\"4 / 3\",\"alt\":\"Dexterous hands · GPU installation\"}]\n24:[\"$\",\"$La\",\"hands_sunglasses_5x\",{\"src\":\"./videos/hands_sunglasses_5x.mp4\",\"poster\":\"./videos/hands_sunglasses_5x.jpg\",\"caption\":\"Sharpa hand picking up and unfolding sunglasses then putting them on a face\",\"ratio\":\"4 / 3\",\"alt\":\"Dexterous hands · sunglasses, 5× speed\"}]\n25:[\"$\",\"$La\",\"dexmate_sharpa_pizza_and_dishes\",{\"src\":\"./videos/dexmate_sharpa_pizza_and_dishes.mp4\",\"poster\":\"./videos/dexmate_sharpa_pizza_and_dishes.jpg\",\"caption\":\"A Dexmate mobile robot with dual Sharpa dexterous hands serving a pizza slice onto a plate and clearing the dishes to the sink\",\"ratio\":\"4 / 3\",\"alt\":\"Kitchen · Dexmate dual Sharpa\"}]\n26:[\"$\",\"$La\",\"studio_sunglasses_noir\",{\"src\":\"./videos/studio_sunglasses_noir.mp4\",\"poster\":\"./videos/studio_sunglasses_noir.jpg\",\"caption\":\"Two Sharpa dexterous hands lifting and unfolding a pair of sunglasses, then setting them on a bust\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · Sharpa sunglasses, noir\"}]\n27:[\"$\",\"$La\",\"sharpa_hello_world_lift_then_uncap\",{\"src\":\"./videos/sharpa_hello_world_lift_then_uncap.mp4\",\"poster\":\"./videos/sharpa_hello_world_lift_then_uncap.jpg\",\"caption\":\"Dual robot arms lifting a marker into a Sharpa dexterous hand, uncapping it, and writing Hello World on a whiteboard\",\"ratio\":\"4 / 3\",\"alt\":\"Studio · Sharpa lift and uncap\"}]\n28:[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"3.3\"}]]}]\n29:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[[\"$\",\"p\",null,{\"children\":\"The generated environments can be adapted for policy evaluation and training for multiple embodiments, and we believe that they will be especially useful for training dextrous hands. Whereas manipulators like the UMI gripper on the Franka Emika Panda can benefit from more accurate physics properties on objects in simulation, dextrous hands must learn to accomplish much more complex tasks with deformable fabrics, plastics, sticky surfaces, and similar challenges. We hope in future simulations to have a pipeline with increased accuracy around specifically supporting packing in plastic bags and applying stickers to diverse surfaces.\"}],[\"$\",\"p\",null,{\"children\":\"Alternative methods of course may use generative diffusion- or autogressive-based world models to generate the multi-view video used for policy evaluation and finetuning. Our method may be beneficial because it is much less computationally intensive than methods relying on generative video models and can be precise for specific environments or parts.\"}]]}]\n2a:[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":1},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-2\",{\"x\":2,\"y\":0,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"1-4\",{\"x\":4,\"y\":1,\"width\":5,\"height\":1}]]}]]}]\n2b:[\"$\",\"text\",null,{\"x\":129,\"y\":404,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"02\"}]\n2c:[\"$\",\"text\",null,{\"x\":173,\"y\":404,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Ingest\"}]\n2d:[\"$\",\"text\",\"0\",{\"x\":14,\"y\":459,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Process input RG"])</script><script>self.__next_f.push([1,"B or other data sources\"}]\n2e:[\"$\",\"text\",\"1\",{\"x\":14,\"y\":485,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"to 3DGS\"}]\n2f:[\"$\",\"text\",null,{\"x\":14,\"y\":514,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-INGEST-DATA\"}]\n30:[\"$\",\"line\",null,{\"x1\":240,\"y1\":528,\"x2\":240,\"y2\":556,\"stroke\":\"var(--text-label)\"}]\n"])</script><script>self.__next_f.push([1,"31:[\"$\",\"g\",\"03\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"03 Step runner: Process the tools for pipeline with visual review of the results, including object masks, semantic features, physics properties, background crop and infill, etc\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":556.5,\"width\":479,\"height\":217,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 572) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":2},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-4\",{\"x\":4,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"0-6\",{\"x\":6,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"0-8\",{\"x\":8,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"1-4\",{\"x\":4,\"y\":1,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"1-6\",{\"x\":6,\"y\":1,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"1-8\",{\"x\":8,\"y\":1,\"width\":1,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":598,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"03\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":598,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Step runner\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":653,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Process the tools for pipeline with visual\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":679,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"review of the results, including object\"}],[\"$\",\"text\",\"2\",{\"x\":14,\"y\":705,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"masks, semantic features, physics\"}],[\"$\",\"text\",\"3\",{\"x\":14,\"y\":731,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"properties, background crop and infill, etc\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":760,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-RUN-STEP\"}]]}],false]}]\n"])</script><script>self.__next_f.push([1,"32:[\"$\",\"div\",null,{\"className\":\"spread__gutter\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"svg\",null,{\"viewBox\":\"0 0 144 932\",\"preserveAspectRatio\":\"none\",\"children\":[\"$\",\"polyline\",null,{\"points\":\"0,665 72,665 72,251 144,251\",\"fill\":\"none\",\"stroke\":\"var(--text-label)\",\"vectorEffect\":\"non-scaling-stroke\"}]}]}]\n"])</script><script>self.__next_f.push([1,"33:[\"$\",\"$1\",\"01\",{\"children\":[[\"$\",\"svg\",null,{\"className\":\"spread__col\",\"viewBox\":\"0 0 480 932\",\"style\":{\"fontFamily\":\"Helvetica, \\\"Helvetica Neue\\\", Arimo, Arial, sans-serif\"},\"children\":[[\"$\",\"g\",null,{\"children\":[[\"$\",\"text\",null,{\"x\":0,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"01\"}],[\"$\",\"text\",null,{\"x\":46,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Conversion\"}],[\"$\",\"line\",null,{\"x1\":0,\"y1\":38.5,\"x2\":480,\"y2\":38.5,\"stroke\":\"var(--ink)\"}],[[\"$\",\"text\",\"0\",{\"x\":0,\"y\":64,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"Segment and predict properties for\"}],[\"$\",\"text\",\"1\",{\"x\":0,\"y\":92,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"objects in scan, then package and inspect\"}]]]}],[[\"$\",\"g\",\"04\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"04 Alignment: Scale, place and rotate each asset by photometric, silhouette and edge residuals\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":168.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 184) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":3},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-2\",{\"x\":2,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"0-4\",{\"x\":4,\"y\":0,\"width\":5,\"height\":1}],[\"$\",\"rect\",\"0-10\",{\"x\":10,\"y\":0,\"width\":1,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"04\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Alignment\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":265,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Scale, place and rotate each asset by\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":291,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"photometric, silhouette and edge residuals\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":320,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-ALIGN-SCENE\"}]]}],[\"$\",\"line\",null,{\"x1\":240,\"y1\":334,\"x2\":240,\"y2\":362,\"stroke\":\"var(--text-label)\"}]]}],[\"$\",\"g\",\"05\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"05 Orientation: Isaac Lab sweeps that orient the completed background 3DGS to the exported scene\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":362.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 378) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":4},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-5\",{\"x\":5,\"y\":0,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"1-6\",{\"x\":6,\"y\":1,\"width\":1,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":404,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"05\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":404,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Orientation\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":459,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Isaac Lab sweeps that orient the completed\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":485,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"background 3DGS to the exported scene\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":514,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-ORIENT-BACKGROUND\"}]]}],[\"$\",\"line\",null,{\"x1\":240,\"y1\":528,\"x2\":240,\"y2\":556,\"stroke\":\"var(--text-label)\"}]]}],[\"$\",\"g\",\"06\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"06 Packager: Publishes the data_processed package for the editor as USDZ\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":556.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 572) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":5},\"children\":[\"$L46\",\"$L47\",\"$L48\"]}]}],\"$L49\",\"$L4a\",[\"$L4b\",\"$L4c\"],\"$L4d\"]}],\"$L4e\"]}],\"$L4f\"]]}],\"$L50\"]}]\n"])</script><script>self.__next_f.push([1,"34:[\"$\",\"$1\",\"02\",{\"children\":[[\"$\",\"svg\",null,{\"className\":\"spread__col\",\"viewBox\":\"0 0 480 932\",\"style\":{\"fontFamily\":\"Helvetica, \\\"Helvetica Neue\\\", Arimo, Arial, sans-serif\"},\"children\":[[\"$\",\"g\",null,{\"children\":[[\"$\",\"text\",null,{\"x\":0,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"02\"}],[\"$\",\"text\",null,{\"x\":46,\"y\":26,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Sisters\"}],[\"$\",\"line\",null,{\"x1\":0,\"y1\":38.5,\"x2\":480,\"y2\":38.5,\"stroke\":\"var(--ink)\"}],[[\"$\",\"text\",\"0\",{\"x\":0,\"y\":64,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"From the fully processed digital twin,\"}],[\"$\",\"text\",\"1\",{\"x\":0,\"y\":92,\"fontSize\":24,\"fill\":\"var(--text-muted)\",\"children\":\"generate a family of sisters\"}]]]}],[[\"$\",\"g\",\"08\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"08 Object sisters: Generate small variations of each object in the scene with generative 3d models, focusing on geometric and visual differences\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":168.5,\"width\":479,\"height\":191,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 184) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":7},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-3\",{\"x\":3,\"y\":0,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"0-8\",{\"x\":8,\"y\":0,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"1-3\",{\"x\":3,\"y\":1,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"1-8\",{\"x\":8,\"y\":1,\"width\":2,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"08\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":210,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Object sisters\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":265,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Generate small variations of each object\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":291,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"in the scene with generative 3d models,\"}],[\"$\",\"text\",\"2\",{\"x\":14,\"y\":317,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"focusing on geometric and visual differences\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":346,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-OBJECT-SISTERS\"}]]}],[\"$\",\"line\",null,{\"x1\":240,\"y1\":360,\"x2\":240,\"y2\":388,\"stroke\":\"var(--text-label)\"}]]}],[\"$\",\"g\",\"09\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"09 Scene sisters: Generate different layouts and arrangement of the sister objects and export them as separate USDZ scenes\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":388.5,\"width\":479,\"height\":191,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 404) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":8},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-3\",{\"x\":3,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"0-9\",{\"x\":9,\"y\":0,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"1-2\",{\"x\":2,\"y\":1,\"width\":9,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":430,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"09\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":430,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Scene sisters\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":485,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Generate different layouts and arrangement\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":511,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"of the sister objects and export them as\"}],[\"$\",\"text\",\"2\",{\"x\":14,\"y\":537,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"separate USDZ scenes\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":566,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-SCENE-SISTERS\"}]]}],false]}]]]}],\"$undefined\"]}]\n"])</script><script>self.__next_f.push([1,"35:[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--ink)\"},\"children\":\"Figure 4 · An overview of the full agentic pipeline with various tools and skills for processing 3d capture data into usable simulation scenes\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"4.2\"}]]}]\n36:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[\"$\",\"p\",null,{\"children\":\"Our agentic workflow processes 3D capture data of diverse origins into fully usable simulation scenes in Isaac Lab, MuJoCo, or Unreal Engine. The agents are helpful in this case specifically because of the diversity of data formats of 3d data and error-prone nature of object conversion and decomposition models. For example, a common failure mode is that the 3d object segmentation will incorrectly transform the scale, rotation, or placement of the objects in the scene, and agents can take camera renders from within Isaac Lab to visually inspect the conversion results and rapidly correct such small errors.\"}]}]\n"])</script><script>self.__next_f.push([1,"37:[\"$\",\"div\",null,{\"className\":\"chapter__exhibit chapter__exhibit--bleed\",\"data-bleed\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(3, minmax(0, 1fr))\"},\"children\":[[\"$\",\"$La\",\"objects/sony_radio_clean\",{\"src\":\"./videos/objects/sony_radio_clean.mp4\",\"poster\":\"./videos/objects/sony_radio_clean.jpg\",\"caption\":\"Sony CFS-43, photogrammetry of a radio cassette player and articulations of its door, keys, knobs, and selector\",\"ratio\":\"4 / 3\",\"alt\":\"Sony radio · full control articulation\"}],[\"$\",\"$La\",\"stegosaurus_walking\",{\"src\":\"./videos/stegosaurus_walking.mp4\",\"poster\":\"./videos/stegosaurus_walking.jpg\",\"caption\":\"A 3dgs of a stegosaurus skeleton from the AMNH, fully articulated and walking (without hand-created animations)\",\"ratio\":\"4 / 3\",\"alt\":\"Stegosaurus · walking\"}],[\"$\",\"$La\",\"objects/ge90_engine_orbit_zoom\",{\"src\":\"./videos/objects/ge90_engine_orbit_zoom.mp4\",\"poster\":\"./videos/objects/ge90_engine_orbit_zoom.jpg\",\"caption\":\"A jet engine with processed turbofan, articulated and running based on engine schematics\",\"ratio\":\"4 / 3\",\"alt\":\"Jet engine · orbit and detail zoom\"}],[\"$\",\"$La\",\"objects/sceneagent_bencini_camera\",{\"src\":\"./videos/objects/sceneagent_bencini_camera.mp4\",\"poster\":\"./videos/objects/sceneagent_bencini_camera.jpg\",\"caption\":\"A Bencini cine camera converted into a usable simulation asset, with usable frame-rate dial and battery compartment\",\"ratio\":\"4 / 3\",\"alt\":\"Bencini cine camera · orbit and detail zoom\"}],[\"$\",\"$La\",\"objects/sceneagent_cessna_cockpit\",{\"src\":\"./videos/objects/sceneagent_cessna_cockpit.mp4\",\"poster\":\"./videos/objects/sceneagent_cessna_cockpit.jpg\",\"caption\":\"A Cessna cockpit, converted into simulation asset with extreme detail in its instrument panel, switches, and placards\",\"ratio\":\"4 / 3\",\"alt\":\"Cessna cockpit · orbit and detail zoom\"}],[\"$\",\"$La\",\"objects/sceneagent_motorcycle\",{\"src\":\"./videos/objects/sceneagent_motorcycle.mp4\",\"poster\":\"./videos/objects/sceneagent_motorcycle.jpg\",\"caption\":\"A motorcycle converted from 3d capture into a usable simulation asset, including many details\",\"ratio\":\"4 / 3\",\"alt\":\"Motorcycle · orbit and detail zoom\"}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"4.3\"}]]}]]}]\n"])</script><script>self.__next_f.push([1,"38:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[\"$\",\"p\",null,{\"children\":[\"One of the most unique and beneficial aspects about our agentic workflow is that it can support converting multiple different data types from diverse sources with useful results. For example, it can similarly process input RGB images and align them with structure-from-motion with colmap and then train a 3DGS scene with NerfStudio’s \",[\"$\",\"code\",null,{\"children\":\"splatfacto\"}],\" big and inspect, review, and correct any results, such as the per image color calibration of the input images. The same pipeline will equally process public internet 3d capture data from photogrammetry or pre-existing 3DGS .ply files.\"]}]}]\n"])</script><script>self.__next_f.push([1,"39:[\"$\",\"div\",null,{\"className\":\"chapter__exhibit chapter__exhibit--bleed\",\"data-bleed\":\"\",\"children\":[[\"$\",\"div\",null,{\"className\":\"clips\",\"style\":{\"gridTemplateColumns\":\"repeat(3, minmax(0, 1fr))\"},\"children\":[[\"$\",\"$La\",\"deformables_textile_rack_orbit\",{\"src\":\"./videos/deformables_textile_rack_orbit.mp4\",\"poster\":\"./videos/deformables_textile_rack_orbit.jpg\",\"caption\":\"A patterned textile draped over a garment rack, orbited to read the folds from every side\",\"ratio\":\"16 / 9\",\"alt\":\"Textile rack · orbit\"}],[\"$\",\"$La\",\"deformables_hoodie_lift_drape\",{\"src\":\"./videos/deformables_hoodie_lift_drape.mp4\",\"poster\":\"./videos/deformables_hoodie_lift_drape.jpg\",\"caption\":\"A hoodie lifted from a surface and left to drape under gravity\",\"ratio\":\"16 / 9\",\"alt\":\"Hoodie · lift and drape\"}],[\"$\",\"$La\",\"deformables_towel_rack_orbit\",{\"src\":\"./videos/deformables_towel_rack_orbit.mp4\",\"poster\":\"./videos/deformables_towel_rack_orbit.jpg\",\"caption\":\"Three towels hung on a wall rack, orbited to read the folds from every side\",\"ratio\":\"16 / 9\",\"alt\":\"Towel rack · orbit\"}],[\"$\",\"$La\",\"deformables_firefighter_hose_unwind\",{\"src\":\"./videos/deformables_firefighter_hose_unwind.mp4\",\"poster\":\"./videos/deformables_firefighter_hose_unwind.jpg\",\"caption\":\"A coiled firefighter hose unwinding under soft-body dynamics\",\"ratio\":\"16 / 9\",\"alt\":\"Firefighter hose · unwind\"}],[\"$\",\"$La\",\"liquid_death_hat_physics\",{\"src\":\"./videos/liquid_death_hat_physics.mp4\",\"poster\":\"./videos/liquid_death_hat_physics.jpg\",\"caption\":\"A Liquid Death corduroy cap with soft-body physics, deforming under contact dynamics\",\"ratio\":\"16 / 9\",\"alt\":\"Liquid Death hat · physics\"}],[\"$\",\"$La\",\"deformables_extension_cord_unwind\",{\"src\":\"./videos/deformables_extension_cord_unwind.mp4\",\"poster\":\"./videos/deformables_extension_cord_unwind.jpg\",\"caption\":\"An extension cord uncoiling from its wound state\",\"ratio\":\"16 / 9\",\"alt\":\"Extension cord · unwind\"}]]}],[\"$\",\"div\",null,{\"className\":\"chapter__caption\",\"children\":[\"$undefined\",[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"var(--font-sans)\",\"fontSize\":10,\"fontWeight\":700,\"lineHeight\":1.5,\"letterSpacing\":\"0.02em\",\"color\":\"var(--accent)\",\"marginLeft\":\"auto\",\"flexShrink\":0},\"children\":\"4.4\"}]]}]]}]\n"])</script><script>self.__next_f.push([1,"3a:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[\"$\",\"p\",null,{\"children\":\"Likewise, it is able to turn synthetic datasets from generative 3DGS models (such as World Labs Marble, NVIDIA Lyra, or TenCent Hy-World). One can prompt the pipeline to convert the full scene into segmented 3d objects with predictive physics properties or instead focus only foreground objects. The first is much more time-consuming and error prone at this point, but we believe that future iterations of the pipeline will easily improve this.\"}]}]\n3b:[\"$\",\"div\",null,{\"className\":\"chapter__text\",\"children\":[\"$\",\"p\",null,{\"children\":\"We imagine a future application of this agentic workflow running on individual in-home or in-factory robots that can 3d capture a space, process that space on-device only with an NVIDIA Jetson Orin or Thor or equivalent, and then train a finetune of its policy on the results with no data leaving device. We hope that this will continue the use of 3DGS in the future to be part of the solution to the data problem of generalist robotic policy development in the future.\"}]}]\n3c:[\"$\",\"p\",null,{\"children\":\"We are still evaluating the results and are optimistic about the results of real world performance improvements after training in converted 3DGS scenes to use in simulation with predictive physics. We will update our work with the full results soon.\"}]\n3d:[\"$\",\"li\",\"motorcycle\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",10,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/e762d944-86f8-405e-8cf7-a7872c4097ba\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Motorcycle\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n3e:[\"$\",\"li\",\"towel-rack\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",11,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/0c8cabee-75cf-454e-864a-8c27ef91f41f\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Towel rack\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n3f:[\"$\",\"li\",\"red-towel\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",12,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/c78e1538-2c3f-4486-ad01-f8eb97098a8a\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Red towel\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n40:[\"$\",\"li\",\"hoodie\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",13,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/33170b4a-c8d2-43ef-82d6-9d5c442c7c6a\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Dept Artisté anorak\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n41:[\"$\",\"li\",\"hose\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",14,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/099a56df-bdb6-429f-89cf-91f1a9bcbee8\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Firefighter hose\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n42:[\"$\",\"li\",\"cord\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",15,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/72007c87-fa9d-41db-bff6-05e230f68cf7\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Electric extension lead\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n43:[\"$\",\"li\",\"hat\",{\"children\":[[\"$\",\"b\",null,{\"children\":[\"[\",16,\"]\"]}],[\"$\",\"span\",null,{\"children\":[\"\",\"“\",[\"$\",\"a\",null,{\"href\":\"https://poly.cam/capture/ad4a76f3-2078-48df-81e2-1460cadae520\",\"target\":\"_blank\",\"rel\":\"noopener noreferrer\",\"children\":\"Liquid Death hat\"}],\",” \",\"Polycam\",\"\",\".\"]}]]}]\n"])</script><script>self.__next_f.push([1,"46:[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}]\n47:[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}]\n48:[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-4\",{\"x\":4,\"y\":0,\"width\":5,\"height\":1}],[\"$\",\"rect\",\"1-4\",{\"x\":4,\"y\":1,\"width\":5,\"height\":1}]]}]\n49:[\"$\",\"text\",null,{\"x\":129,\"y\":598,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"06\"}]\n4a:[\"$\",\"text\",null,{\"x\":173,\"y\":598,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Packager\"}]\n4b:[\"$\",\"text\",\"0\",{\"x\":14,\"y\":653,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Publishes the data_processed package for\"}]\n4c:[\"$\",\"text\",\"1\",{\"x\":14,\"y\":679,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"the editor as USDZ\"}]\n4d:[\"$\",\"text\",null,{\"x\":14,\"y\":708,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-PACKAGE-DATA-PROCESSED\"}]\n4e:[\"$\",\"line\",null,{\"x1\":240,\"y1\":722,\"x2\":240,\"y2\":750,\"stroke\":\"var(--text-label)\"}]\n"])</script><script>self.__next_f.push([1,"4f:[\"$\",\"g\",\"07\",{\"children\":[[\"$\",\"g\",null,{\"className\":\"agent-node\",\"role\":\"img\",\"aria-label\":\"07 Inspector: Visually reviews and fixes anything off in the final prepared USDZ in NVIDIA Isaac Lab\",\"children\":[[\"$\",\"rect\",null,{\"x\":0.5,\"y\":750.5,\"width\":479,\"height\":165,\"fill\":\"var(--paper)\",\"stroke\":\"var(--ink)\"}],[\"$\",\"g\",null,{\"transform\":\"translate(14 766) scale(7)\",\"fill\":\"currentColor\",\"shapeRendering\":\"crispEdges\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"g\",null,{\"className\":\"agent-sprite\",\"style\":{\"--i\":6},\"children\":[[\"$\",\"g\",\"leg\",{\"className\":\"agent-sprite__leg\",\"children\":[[\"$\",\"rect\",\"6-4\",{\"x\":4,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-6\",{\"x\":6,\"y\":6,\"width\":1,\"height\":1}],[\"$\",\"rect\",\"6-8\",{\"x\":8,\"y\":6,\"width\":1,\"height\":1}]]}],[\"$\",\"g\",\"body\",{\"className\":\"agent-sprite__body\",\"children\":[[\"$\",\"rect\",\"3-2\",{\"x\":2,\"y\":3,\"width\":9,\"height\":1}],[\"$\",\"rect\",\"4-2\",{\"x\":2,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"4-5\",{\"x\":5,\"y\":4,\"width\":3,\"height\":1}],[\"$\",\"rect\",\"4-9\",{\"x\":9,\"y\":4,\"width\":2,\"height\":1}],[\"$\",\"rect\",\"5-2\",{\"x\":2,\"y\":5,\"width\":9,\"height\":1}]]}],[\"$\",\"g\",\"tool\",{\"className\":\"agent-sprite__tool\",\"children\":[[\"$\",\"rect\",\"0-4\",{\"x\":4,\"y\":0,\"width\":4,\"height\":1}],[\"$\",\"rect\",\"1-4\",{\"x\":4,\"y\":1,\"width\":2,\"height\":1}]]}]]}]}],[\"$\",\"text\",null,{\"x\":129,\"y\":792,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--accent)\",\"children\":\"07\"}],[\"$\",\"text\",null,{\"x\":173,\"y\":792,\"fontSize\":26,\"fontWeight\":700,\"fill\":\"var(--ink)\",\"children\":\"Inspector\"}],[[\"$\",\"text\",\"0\",{\"x\":14,\"y\":847,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"Visually reviews and fixes anything off in\"}],[\"$\",\"text\",\"1\",{\"x\":14,\"y\":873,\"fontSize\":21,\"fill\":\"var(--text-body)\",\"children\":\"the final prepared USDZ in NVIDIA Isaac Lab\"}]],[\"$\",\"text\",null,{\"x\":14,\"y\":902,\"fontSize\":15,\"fontFamily\":\"var(--font-pixel)\",\"letterSpacing\":\"0.08em\",\"fill\":\"var(--text-label)\",\"children\":\"REAL2SIM-INSPECT-ISAAC\"}]]}],false]}]\n"])</script><script>self.__next_f.push([1,"50:[\"$\",\"div\",null,{\"className\":\"spread__gutter\",\"aria-hidden\":\"true\",\"children\":[\"$\",\"svg\",null,{\"viewBox\":\"0 0 144 932\",\"preserveAspectRatio\":\"none\",\"children\":[\"$\",\"polyline\",null,{\"points\":\"0,833 72,833 72,264 144,264\",\"fill\":\"none\",\"stroke\":\"var(--text-label)\",\"vectorEffect\":\"non-scaling-stroke\"}]}]}]\n"])</script><script>self.__next_f.push([1,"1e:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"meta\",\"1\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}]]\n"])</script><script>self.__next_f.push([1,"21:[[\"$\",\"title\",\"0\",{\"children\":\"SceneAgent | Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"1\",{\"name\":\"description\",\"content\":\"SceneAgent converts 3D captures of real-world scenes into simulation environments with predictive physics for robot policy evaluation, planning and training.\"}],[\"$\",\"link\",\"2\",{\"rel\":\"author\",\"href\":\"https://computationalrobotics.seas.harvard.edu/\"}],[\"$\",\"meta\",\"3\",{\"name\":\"author\",\"content\":\"Luke Hollis\"}],[\"$\",\"link\",\"4\",{\"rel\":\"author\",\"href\":\"https://computationalrobotics.seas.harvard.edu/\"}],[\"$\",\"meta\",\"5\",{\"name\":\"author\",\"content\":\"Tianxing Fan\"}],[\"$\",\"link\",\"6\",{\"rel\":\"author\",\"href\":\"https://computationalrobotics.seas.harvard.edu/\"}],[\"$\",\"meta\",\"7\",{\"name\":\"author\",\"content\":\"Heng Yang\"}],[\"$\",\"meta\",\"8\",{\"name\":\"keywords\",\"content\":\"SceneAgent,real-to-sim,sim-to-real,3D Gaussian Splatting,simulation environments,predictive physics,robot policy evaluation,policy training,online planning,object segmentation,articulation,deformable objects,digital sisters,robot learning,Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"9\",{\"name\":\"creator\",\"content\":\"Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"10\",{\"name\":\"publisher\",\"content\":\"Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"11\",{\"name\":\"robots\",\"content\":\"index, follow\"}],[\"$\",\"meta\",\"12\",{\"name\":\"googlebot\",\"content\":\"index, follow, max-video-preview:-1, max-image-preview:large, max-snippet:-1\"}],[\"$\",\"meta\",\"13\",{\"name\":\"citation_title\",\"content\":\"SceneAgent: 3D Capture-Derived Scenes with Predictive Physics for Policy Evaluation and Training Environments\"}],[\"$\",\"meta\",\"14\",{\"name\":\"citation_author\",\"content\":\"Luke Hollis\"}],[\"$\",\"meta\",\"15\",{\"name\":\"citation_author\",\"content\":\"Tianxing Fan\"}],[\"$\",\"meta\",\"16\",{\"name\":\"citation_author\",\"content\":\"Heng Yang\"}],[\"$\",\"meta\",\"17\",{\"name\":\"citation_publication_date\",\"content\":\"2026\"}],[\"$\",\"meta\",\"18\",{\"name\":\"citation_publisher\",\"content\":\"Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"19\",{\"name\":\"citation_abstract_html_url\",\"content\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/\"}],[\"$\",\"link\",\"20\",{\"rel\":\"canonical\",\"href\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/\"}],[\"$\",\"meta\",\"21\",{\"property\":\"og:title\",\"content\":\"SceneAgent: 3D captures to simulation environments with predictive physics\"}],[\"$\",\"meta\",\"22\",{\"property\":\"og:description\",\"content\":\"An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint.\"}],[\"$\",\"meta\",\"23\",{\"property\":\"og:url\",\"content\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/\"}],[\"$\",\"meta\",\"24\",{\"property\":\"og:site_name\",\"content\":\"Harvard Computational Robotics Group\"}],[\"$\",\"meta\",\"25\",{\"property\":\"og:locale\",\"content\":\"en_US\"}],[\"$\",\"meta\",\"26\",{\"property\":\"og:image\",\"content\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg\"}],[\"$\",\"meta\",\"27\",{\"property\":\"og:image:secure_url\",\"content\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg\"}],[\"$\",\"meta\",\"28\",{\"property\":\"og:image:type\",\"content\":\"image/jpeg\"}],[\"$\",\"meta\",\"29\",{\"property\":\"og:image:width\",\"content\":\"1200\"}],[\"$\",\"meta\",\"30\",{\"property\":\"og:image:height\",\"content\":\"630\"}],[\"$\",\"meta\",\"31\",{\"property\":\"og:image:alt\",\"content\":\"SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle\"}],[\"$\",\"meta\",\"32\",{\"property\":\"og:type\",\"content\":\"website\"}],[\"$\",\"meta\",\"33\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"34\",{\"name\":\"twitter:title\",\"content\":\"SceneAgent: 3D captures to simulation environments with predictive physics\"}],[\"$\",\"meta\",\"35\",{\"name\":\"twitter:description\",\"content\":\"An agentic pipeline from 3D Gaussian Splatting, photogrammetry and LiDAR captures of real scenes to simulation environments with predictive physics. Harvard Computational Robotics Group, 2026 preprint.\"}],\"$L51\",\"$L52\",\"$L53\",\"$L54\"]\n"])</script><script>self.__next_f.push([1,"45:null\n"])</script><script>self.__next_f.push([1,"55:I[29447,[\"./_next/static/chunks/3d-g45tvvtwds.js\"],\"IconMark\"]\n51:[\"$\",\"meta\",\"36\",{\"name\":\"twitter:image\",\"content\":\"https://computationalrobotics.seas.harvard.edu/SceneAgent/branding/sceneagent-social-share.jpg\"}]\n52:[\"$\",\"meta\",\"37\",{\"name\":\"twitter:image:alt\",\"content\":\"SceneAgent: two Franka robot arms over a simulated tabletop scene, above the paper title and subtitle\"}]\n53:[\"$\",\"link\",\"38\",{\"rel\":\"icon\",\"href\":\"./favicon.png\",\"type\":\"image/png\"}]\n54:[\"$\",\"$L55\",\"39\",{}]\n"])</script></body></html>