2 lines
No EOL
13 KiB
HTML
2 lines
No EOL
13 KiB
HTML
<!doctype html><html data-theme=light lang=en xmlns=http://www.w3.org/1999/xhtml><head><meta charset=UTF-8><meta name=description><meta content="width=device-width,initial-scale=1" name=viewport><meta content=#FF7E3C name=theme-color><meta content=#CC5A26 media=(prefers-color-scheme:dark) name=theme-color><title>Stable Dreamfusion - Aron Petau</title><link href=https://aron.petau.net/project/stable-dreamfusion/ rel=canonical><link href=https://mastodon.online/@reprintedAron rel=me><meta content=@reprintedAron@mastodon.online name=fediverse:creator><link href=/favicon/favicon-96x96.png rel=icon sizes=96x96 type=image/png><link href=/favicon/favicon.svg rel=icon type=image/svg+xml><link rel="shortcut icon" href=/favicon/favicon.ico><link href=/favicon/apple-touch-icon.png rel=apple-touch-icon sizes=180x180><meta content="Aron Petau" name=apple-mobile-web-app-title><link href=/favicon/site.webmanifest rel=manifest><link crossorigin href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css integrity=sha512-... referrerpolicy=no-referrer rel=stylesheet><link title="Aron Petau - RSS Feed" href=https://aron.petau.net/rss.xml rel=alternate type=application/rss+xml><link title="Aron Petau - Atom Feed" href=https://aron.petau.net/atom.xml rel=alternate type=application/atom+xml><style>:root{--accent-color:#ff7e3c}[data-theme=dark]{--accent-color:#cc5a26}@media (prefers-color-scheme:dark){:root:not([data-theme=light]){--accent-color:#cc5a26}}</style><link href=https://aron.petau.net/style.css rel=stylesheet><link href=https://aron.petau.net/css/timeline.css rel=stylesheet><link href=https://aron.petau.net/css/mermaid.css rel=stylesheet><link href=https://aron.petau.net/css/skills.css rel=stylesheet><link href=https://aron.petau.net/css/gallery.css rel=stylesheet><script defer src=https://aron.petau.net/closable.js></script><script defer src=https://aron.petau.net/copy-button.js></script><script data-goatcounter=https://awebsite.goatcounter.com/count defer src=https://aron.petau.net/count.js></script><script defer src=https://aron.petau.net/fuse.js></script><script defer src=https://aron.petau.net/search-fuse.js></script><script defer src=https://aron.petau.net/theme-switcher.js></script><script type=module>import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs';
|
|
mermaid.initialize({ startOnLoad: true });</script><meta content="Aron Petau" property=og:site_name><meta content="Stable Dreamfusion - Aron Petau" property=og:title><meta content=https://aron.petau.net/project/stable-dreamfusion/ property=og:url><meta content="An exploration of 3D mesh generation through AI" property=og:description><meta content=https://aron.petau.net/card.png property=og:image><meta content=en_US property=og:locale><body><header id=site-nav><nav><a href=#main-content tabindex=0> Skip to Main Content </a><ul><li id=home><a href=https://aron.petau.net> <i class=icon></i>Aron Petau</a><li class=divider><li><a href=https://aron.petau.net/project/>Projects</a><li><a href=https://aron.petau.net/pages/contact/>Contact</a><li><a href=https://aron.petau.net/pages/cv/>CV</a><li><a href=https://aron.petau.net/pages/about/>About</a><li id=search><button class=circle id=search-toggle title=Search><i class=icon></i></button><li id=language-switcher><details class=closable><summary class=circle title=Language><i class=icon></i></summary> <ul><li><a href=https://aron.petau.net//de/project/stable-dreamfusion/ lang=de>Deutsch</a></ul></details><li id=theme-switcher><details class=closable><summary class=circle title=Theme><i class=icon></i></summary> <ul><li><button title="Switch to Light Theme" class=circle id=theme-light><i class=icon></i></button><li><button title="Switch to Dark Theme" class=circle id=theme-dark><i class=icon></i></button><li><button title="Use System Theme" class=circle id=theme-system><i class=icon></i></button></ul></details><li id=feed><details class=closable><summary class=circle title=Feed><i class=icon></i></summary> <ul><li><a href=https://aron.petau.net/rss.xml>RSS</a><li><a href=https://aron.petau.net/atom.xml>Atom</a></ul></details><li id=repo><a class=circle href=https://forgejo.petau.net/aron/awebsite title=Repository> <i class=icon></i> </a></ul></nav><div id=search-container><label class=visually-hidden for=search-bar>Search</label><input placeholder="Search for…" autocomplete=off disabled id=search-bar type=search><div id=search-results-container><div id=search-results></div></div></div></header><main id=main-content><article><div id=heading><p><small> <time datetime=" 2023-06-20T00:00:00+00:00">Published on June 20, 2023</time></small><h1>Stable Dreamfusion</h1><p><small><span>By Aron Petau</span><span> • </span><span>2 minutes read</span><span> • </span></small><ul class=tags><li><a class=tag href=https://aron.petau.net/tags/3d-graphics/>3d graphics</a><li><a class=tag href=https://aron.petau.net/tags/ai/>ai</a><li><a class=tag href=https://aron.petau.net/tags/dreamfusion/>dreamfusion</a><li><a class=tag href=https://aron.petau.net/tags/generative/>generative</a><li><a class=tag href=https://aron.petau.net/tags/mesh/>mesh</a><li><a class=tag href=https://aron.petau.net/tags/studio-d-c/>studio d+c</a><li><a class=tag href=https://aron.petau.net/tags/university-of-the-arts-berlin/>university of the arts berlin</a></ul></div><div id=buttons-container><a title="Go to Top" href=#top id=go-to-top><i class=icon></i></a><a href="https://shareopenly.org/share/?url=https://aron.petau.net/project/stable-dreamfusion/&text=An%20exploration%20of%203D%20mesh%20generation%20through%20AI" id=share title=Share><i class=icon></i></a><a title="File an Issue" href=https://forgejo.petau.net/aron/awebsite/issues id=issue><i class=icon></i></a></div><h2 id=Stable_Dreamfusion>Stable Dreamfusion</h2><div class=sketchfab-embed-wrapper><iframe allow="autoplay; fullscreen; xr-spatial-tracking" src="https://sketchfab.com/models/0af6d95988e44c73a693c45e1db44cad/embed?ui_theme=dark&dnt=1" title="Stable-Dreamfusion Pig" allowfullscreen execution-while-not-rendered execution-while-out-of-viewport frameborder=0 height=600 mozallowfullscreen=true web-share webkitallowfullscreen=true width=800 xr-spatial-tracking></iframe></div><h2 id=Sources>Sources</h2><p>I forked a popular implementation that reverse-engineered the Google Dreamfusion algorithm. This algorithm is closed-source and not publicly available. You can find my forked implementation <a href=https://github.com/arontaupe/stable-dreamfusion>on my GitHub repository</a>. This version runs on Stable Diffusion as its base process, which means we can expect results that might not match Google's quality. The <a href=https://dreamfusion3d.github.io>original DreamFusion paper and implementation</a> provides more details about the technique.</p><iframe allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" allowfullscreen class=youtube-embed referrerpolicy=strict-origin-when-cross-origin src=https://www.youtube-nocookie.com/embed/shW_Jh728yg></iframe><h2 id=Gradio>Gradio</h2><p>I forked the code to implement my own Gradio interface for the algorithm. Gradio is a great tool for quickly building interfaces for machine learning models. No coding is required for the end user - they can simply state their wish, and the system will generate a ready-to-be-rigged 3D model (OBJ file).<h2 id=Mixamo>Mixamo</h2><p>I used Mixamo to rig the model. It's a powerful tool for rigging and animating models, but its main strength is simplicity. As long as you have a model with a reasonable humanoid shape in a T-pose, you can rig it in seconds. That's exactly what I did here.<h2 id=Unity>Unity</h2><p>I used Unity to render the model for the Magic Leap 1 headset. This allowed me to create an interactive and immersive environment with the generated models.<p>The vision was to build an AI Chamber of Wishes: You put on the AR glasses, state your desires, and the algorithm presents you with an almost-real object in augmented reality.<p>Due to not having access to Google's proprietary source code and the limitations of our studio computers (which, while powerful, aren't quite optimized for machine learning), the results weren't as refined as I had hoped. Nevertheless, the results are fascinating, and I'm satisfied with the outcome. A single object generation in the environment takes approximately 20 minutes. The algorithm can be quite temperamental - it often struggles to generate coherent objects, but when it succeeds, the results are quite impressive.</article><hr><nav id=post-nav><a class="post-nav-item post-nav-prev" href=https://aron.petau.net/project/ascendancy/> <div class=nav-arrow>Previous</div> <span class=post-title>Ascendancy</span> </a><a class="post-nav-item post-nav-next" href=https://aron.petau.net/project/dreams-of-cars/> <div class=nav-arrow>Next</div> <span class=post-title>Dreams of Cars</span> </a></nav><span class=hidden id=copy-code-text>Copy Code</span><span class=hidden id=search-index>https://aron.petau.net/search_index.en.json</span><span class=hidden id=more-matches-text>$MATCHES more matches</span></main><footer id=site-footer><div class=carbonbadge id=wcb></div><script defer src=https://unpkg.com/website-carbon-badges@1.1.3/b.min.js></script><p><nav><ul><li><a href=https://aron.petau.net/project/>Projects</a><li><a href=https://aron.petau.net/pages/privacy/>Privacy</a><li><a class=external href=https://kaesewerkstatt.petau.net>Käsewerkstatt</a><li><a class=external href=https://www.newpractice.net/author/aron-petau>New Practice Network</a></ul></nav><p>© Aron Petau, 2025<p><a class=external href=https://forgejo.petau.net/aron/awebsite>Website source</a><ul id=socials><li><a rel=" me" href=https://github.com/arontaupe title=GitHub> <i style="--icon:url("data:image/svg+xml,%3Csvg role='img' viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Ctitle%3EGitHub%3C/title%3E%3Cpath d='M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12'/%3E%3C/svg%3E")" class=icon></i> <span>GitHub</span> </a><li><a rel=" me" href=https://www.printables.com/@arontaupe title=Printables> <i class=icon style=--icon:url(data:image/svg+xml,%3Csvg%20role%3D%22img%22%20viewBox%3D%220%200%2024%2024%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3Ctitle%3EPrintables%3C%2Ftitle%3E%3Cpath%20d%3D%22M3.678%204.8%2012%209.6v9.6l8.322-4.8V4.8L12%200ZM12%2019.2l-8.322-4.8V24Z%22%2F%3E%3C%2Fsvg%3E)></i> <span>Printables</span> </a><li><a rel=" me" href=https://www.etsy.com/de-en/shop/reprintedservices title=Etsy> <i class=icon style=--icon:url(data:image/svg+xml,%3Csvg%20role%3D%22img%22%20viewBox%3D%220%200%2024%2024%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%3Ctitle%3EEtsy%3C%2Ftitle%3E%3Cpath%20d%3D%22M8.559%202.445c0-.325.033-.52.59-.52h7.465c1.3%200%202.02%201.11%202.54%203.193l.42%201.666h1.27c.23-4.728.43-6.784.43-6.784s-3.196.36-5.09.36H6.635L1.521.196v1.37l1.725.326c1.21.24%201.5.496%201.6%201.606%200%200%20.11%203.27.11%208.64%200%205.385-.09%208.61-.09%208.61%200%20.973-.39%201.333-1.59%201.573l-1.722.33V24l5.13-.165h8.55c1.935%200%206.39.165%206.39.165.105-1.17.75-6.48.855-7.064h-1.2l-1.284%202.91c-1.005%202.28-2.476%202.445-4.11%202.445h-4.906c-1.63%200-2.415-.64-2.415-2.05V12.8s3.62%200%204.79.096c.912.064%201.463.325%201.76%201.598l.39%201.695h1.41l-.09-4.278.192-4.305h-1.391l-.45%201.89c-.283%201.244-.48%201.47-1.754%201.6-1.666.17-4.815.14-4.815.14V2.45h-.05z%22%2F%3E%3C%2Fsvg%3E)></i> <span>Etsy</span> </a><li><a rel=" me" href=https://mastodon.online/@reprintedAron title=Mastodon> <i style="--icon:url("data:image/svg+xml,%3Csvg role='img' viewBox='0 0 24 24' xmlns='http://www.w3.org/2000/svg'%3E%3Ctitle%3EMastodon%3C/title%3E%3Cpath d='M23.268 5.313c-.35-2.578-2.617-4.61-5.304-5.004C17.51.242 15.792 0 11.813 0h-.03c-3.98 0-4.835.242-5.288.309C3.882.692 1.496 2.518.917 5.127.64 6.412.61 7.837.661 9.143c.074 1.874.088 3.745.26 5.611.118 1.24.325 2.47.62 3.68.55 2.237 2.777 4.098 4.96 4.857 2.336.792 4.849.923 7.256.38.265-.061.527-.132.786-.213.585-.184 1.27-.39 1.774-.753a.057.057 0 0 0 .023-.043v-1.809a.052.052 0 0 0-.02-.041.053.053 0 0 0-.046-.01 20.282 20.282 0 0 1-4.709.545c-2.73 0-3.463-1.284-3.674-1.818a5.593 5.593 0 0 1-.319-1.433.053.053 0 0 1 .066-.054c1.517.363 3.072.546 4.632.546.376 0 .75 0 1.125-.01 1.57-.044 3.224-.124 4.768-.422.038-.008.077-.015.11-.024 2.435-.464 4.753-1.92 4.989-5.604.008-.145.03-1.52.03-1.67.002-.512.167-3.63-.024-5.545zm-3.748 9.195h-2.561V8.29c0-1.309-.55-1.976-1.67-1.976-1.23 0-1.846.79-1.846 2.35v3.403h-2.546V8.663c0-1.56-.617-2.35-1.848-2.35-1.112 0-1.668.668-1.67 1.977v6.218H4.822V8.102c0-1.31.337-2.35 1.011-3.12.696-.77 1.608-1.164 2.74-1.164 1.311 0 2.302.5 2.962 1.498l.638 1.06.638-1.06c.66-.999 1.65-1.498 2.96-1.498 1.13 0 2.043.395 2.74 1.164.675.77 1.012 1.81 1.012 3.12z'/%3E%3C/svg%3E")" class=icon></i> <span>Mastodon</span> </a></ul></footer> |