//! Demonstrates the use of our depth cloud renderer, which will efficiently draw a point cloud //! using a depth texture and a set of intrinsics. //! //! ## Usage //! //! Native: //! ```sh //! cargo r -p re_renderer --example depth_cloud //! ``` //! //! Web: //! ```sh //! cargo run-wasm --example depth_cloud //! ``` #![expect(clippy::disallowed_methods)] // allow hardcoded colors use std::f32::consts::TAU; use glam::Vec3; use itertools::Itertools as _; use macaw::IsoTransform; use re_renderer::renderer::{ ColormappedTexture, DepthCloud, DepthCloudDrawData, DepthClouds, DrawData, GenericSkyboxDrawData, RectangleDrawData, RectangleOptions, TexturedRect, }; use re_renderer::resource_managers::{GpuTexture2D, ImageDataDesc}; use re_renderer::view_builder::{self, Projection, ViewBuilder}; use re_renderer::{Color32, LineDrawableBuilder, PointCloudBuilder, Rgba, Size}; use winit::event::ElementState; use winit::keyboard; mod framework; // --- // TODO(#1426): unify camera logic between examples. enum CameraControl { RotateAroundCenter, // TODO(andreas): Only pauses rotation right now. Add camera controller. Manual, } struct RenderDepthClouds { depth: DepthTexture, albedo: AlbedoTexture, scale: f32, point_radius_from_world_depth: f32, intrinsics: glam::Mat3, camera_control: CameraControl, camera_position: glam::Vec3, } impl RenderDepthClouds { /// Manually backproject the depth texture into a point cloud and render it. fn draw_backprojected_point_cloud( &mut self, re_ctx: &re_renderer::RenderContext, pixels_per_point: f32, resolution_in_pixel: [u32; 2], target_location: glam::Vec2, frame_draw_data: FD, image_draw_data: ID, ) -> anyhow::Result where FD: DrawData + Sync + Send + Clone + 'static, ID: DrawData + Sync + Send + Clone + 'static, { let Self { depth, scale, point_radius_from_world_depth, intrinsics, .. } = self; let focal_length = glam::Vec2::new(intrinsics.x_axis.x, intrinsics.y_axis.y); let offset = glam::Vec2::new(intrinsics.z_axis.x, intrinsics.z_axis.y); let point_cloud_draw_data = { let (points, colors, radii): (Vec<_>, Vec<_>, Vec<_>) = (0..depth.dimensions.y) .flat_map(|y| (0..depth.dimensions.x).map(move |x| glam::UVec2::new(x, y))) .map(|texcoords| { let linear_depth = depth.get_linear(texcoords.x, texcoords.y); let pos_in_world = ((texcoords.as_vec2() - offset) * linear_depth / focal_length) .extend(linear_depth); ( pos_in_world * *scale, Color32::from_gray((linear_depth * 255.0) as u8), Size(linear_depth * *point_radius_from_world_depth), ) }) .multiunzip(); let mut builder = PointCloudBuilder::new(re_ctx); builder.batch("backprojected point cloud").add_points_slow( &points, &radii, &colors, &[], ); builder.into_draw_data()? }; let mut view_builder = ViewBuilder::new( re_ctx, view_builder::TargetConfiguration { name: "Point Cloud".into(), resolution_in_pixel, view_from_world: IsoTransform::look_at_rh( self.camera_position, Vec3::ZERO, Vec3::Y, ) .ok_or_else(|| anyhow::format_err!("invalid camera"))?, projection_from_view: Projection::Perspective { vertical_fov: 70.0 * std::f32::consts::TAU / 360.0, near_plane_distance: 0.01, aspect_ratio: resolution_in_pixel[0] as f32 / resolution_in_pixel[1] as f32, }, pixels_per_point, ..Default::default() }, )?; let command_buffer = view_builder .queue_draw( re_ctx, GenericSkyboxDrawData::new(re_ctx, Default::default()), ) .queue_draw(re_ctx, point_cloud_draw_data) .queue_draw(re_ctx, frame_draw_data) .queue_draw(re_ctx, image_draw_data) .draw(re_ctx, re_renderer::Rgba::TRANSPARENT)?; Ok(framework::ViewDrawResult { view_builder, command_buffer, target_location, }) } /// Pass the depth texture to our native depth cloud renderer. fn draw_depth_cloud( &mut self, re_ctx: &re_renderer::RenderContext, pixels_per_point: f32, resolution_in_pixel: [u32; 2], target_location: glam::Vec2, frame_draw_data: FD, image_draw_data: ID, ) -> anyhow::Result where FD: DrawData + Sync + Send + Clone + 'static, ID: DrawData + Sync + Send + Clone + 'static, { let Self { depth, scale, point_radius_from_world_depth, intrinsics, .. } = self; let world_from_rdf = glam::Affine3A::from_scale(glam::Vec3::splat(*scale)); let depth_cloud_draw_data = DepthCloudDrawData::new( re_ctx, &DepthClouds { clouds: vec![DepthCloud { world_from_rdf, depth_camera_intrinsics: *intrinsics, world_depth_from_texture_depth: 1.0, point_radius_from_world_depth: *point_radius_from_world_depth, min_max_depth_in_world: [0.0, 5.0], depth_dimensions: depth.dimensions, depth_texture: depth.texture.clone(), colormap: re_renderer::Colormap::Turbo, outline_mask_id: Default::default(), picking_object_id: Default::default(), }], radius_boost_in_ui_points_for_outlines: 2.5, }, )?; let mut view_builder = ViewBuilder::new( re_ctx, view_builder::TargetConfiguration { name: "Depth Cloud".into(), resolution_in_pixel, view_from_world: IsoTransform::look_at_rh( self.camera_position, Vec3::ZERO, Vec3::Y, ) .ok_or_else(|| anyhow::format_err!("invalid camera"))?, projection_from_view: Projection::Perspective { vertical_fov: 70.0 * std::f32::consts::TAU / 360.0, near_plane_distance: 0.01, aspect_ratio: resolution_in_pixel[0] as f32 / resolution_in_pixel[1] as f32, }, pixels_per_point, ..Default::default() }, )?; let command_buffer = view_builder .queue_draw( re_ctx, GenericSkyboxDrawData::new(re_ctx, Default::default()), ) .queue_draw(re_ctx, depth_cloud_draw_data) .queue_draw(re_ctx, frame_draw_data) .queue_draw(re_ctx, image_draw_data) .draw(re_ctx, re_renderer::Rgba::TRANSPARENT)?; Ok(framework::ViewDrawResult { view_builder, command_buffer, target_location, }) } } impl framework::Example for RenderDepthClouds { fn title() -> &'static str { "Depth clouds" } fn new(re_ctx: &re_renderer::RenderContext) -> Self { re_log::info!("Stop camera movement by pressing 'Space'"); let depth = DepthTexture::spiral(re_ctx, glam::uvec2(640, 480)); let albedo = AlbedoTexture::spiral(re_ctx, depth.dimensions); let scale = 50.0; let point_radius_from_world_depth = 0.1; // hardcoded intrinsics for nyud dataset let focal_length = depth.dimensions.x as f32 * 0.7; let offset = depth.dimensions.as_vec2() * 0.5; let intrinsics = glam::Mat3::from_cols( Vec3::new(focal_length, 0.0, offset.x), Vec3::new(0.0, focal_length, offset.y), Vec3::new(0.0, 0.0, 1.0), ) .transpose(); Self { depth, albedo, scale, point_radius_from_world_depth, intrinsics, camera_control: CameraControl::RotateAroundCenter, camera_position: glam::Vec3::ZERO, } } fn draw( &mut self, re_ctx: &re_renderer::RenderContext, resolution: [u32; 2], time: &framework::Time, pixels_per_point: f32, ) -> anyhow::Result> { let Self { albedo, camera_control, camera_position, .. } = self; let secs_since_startup = time.secs_since_startup(); if matches!(camera_control, CameraControl::RotateAroundCenter) { *camera_position = Vec3::new(secs_since_startup.sin(), 0.5, secs_since_startup.cos()) * 100.0; } let splits = framework::split_resolution(resolution, 1, 2).collect::>(); let frame_size = albedo.dimensions.as_vec2().extend(0.0) / 15.0; let scale = glam::Affine3A::from_scale(frame_size); let rotation = glam::Affine3A::IDENTITY; let translation_center = glam::Affine3A::from_translation(-glam::Vec3::splat(0.5) * frame_size); let world_from_model = rotation * translation_center * scale; let frame_draw_data = { let mut builder = LineDrawableBuilder::new(re_ctx); { let mut line_batch = builder.batch("frame").world_from_obj(world_from_model); line_batch.add_box_outline_from_transform( glam::Affine3A::from_scale_rotation_translation( glam::Vec3::new(1.0, 1.0, 0.0), Default::default(), glam::Vec3::ONE * 0.5, ), ); } builder.into_draw_data()? }; let image_draw_data = RectangleDrawData::new( re_ctx, &[TexturedRect { top_left_corner_position: world_from_model .transform_point3(glam::Vec3::new(1.0, 1.0, 0.0)), extent_u: world_from_model.transform_vector3(-glam::Vec3::X), extent_v: world_from_model.transform_vector3(-glam::Vec3::Y), colormapped_texture: ColormappedTexture::from_unorm_rgba(albedo.texture.clone()), options: RectangleOptions { texture_filter_magnification: re_renderer::renderer::TextureFilterMag::Nearest, texture_filter_minification: re_renderer::renderer::TextureFilterMin::Linear, multiplicative_tint: Rgba::from_white_alpha(0.5), depth_offset: -1, ..Default::default() }, }], )?; Ok(vec![ self.draw_backprojected_point_cloud( re_ctx, pixels_per_point, splits[0].resolution_in_pixel, splits[0].target_location, frame_draw_data.clone(), image_draw_data.clone(), )?, self.draw_depth_cloud( re_ctx, pixels_per_point, splits[1].resolution_in_pixel, splits[1].target_location, frame_draw_data, image_draw_data, )?, ]) } fn on_key_event(&mut self, input: winit::event::KeyEvent) { #![expect(clippy::single_match)] match (input.state, input.logical_key) { (ElementState::Released, keyboard::Key::Named(keyboard::NamedKey::Space)) => { self.camera_control = match self.camera_control { CameraControl::RotateAroundCenter => CameraControl::Manual, CameraControl::Manual => CameraControl::RotateAroundCenter, }; } _ => {} } } } fn main() { framework::start::(); } // --- /// Returns `(position_in_image, linear_depth)`. fn spiral(dimensions: glam::UVec2) -> impl Iterator { let size = (dimensions.x * dimensions.y) as usize; let factor = dimensions.as_vec2() - 1.0; let mut i = 0; let mut angle_rad: f32 = 0.0; std::iter::from_fn(move || { if i < size { let radius = i as f32 / size as f32; let pos = glam::Vec2::splat(0.5) + glam::Vec2::new(angle_rad.cos(), angle_rad.sin()) * 0.5 * radius; let texcoords = (pos * factor).as_uvec2(); i += 1; angle_rad += 0.0005 * TAU; return Some((texcoords, radius)); } None }) } pub fn hash(value: &impl std::hash::Hash) -> u64 { ahash::RandomState::with_seeds(1, 2, 3, 4).hash_one(value) } struct DepthTexture { dimensions: glam::UVec2, data: Vec, texture: GpuTexture2D, } impl DepthTexture { pub fn spiral(re_ctx: &re_renderer::RenderContext, dimensions: glam::UVec2) -> Self { let size = (dimensions.x * dimensions.y) as usize; let mut data = vec![0f32; size]; spiral(dimensions).for_each(|(texcoords, d)| { data[(texcoords.x + texcoords.y * dimensions.x) as usize] = d; }); let label = format!("depth texture spiral {dimensions}"); let texture = re_ctx .texture_manager_2d .get_or_create( hash(&label), re_ctx, ImageDataDesc { label: label.into(), data: bytemuck::cast_slice(&data).into(), format: wgpu::TextureFormat::R32Float.into(), width_height: dimensions.to_array(), alpha_channel_usage: re_renderer::AlphaChannelUsage::Opaque, }, ) .expect("Failed to create depth texture."); Self { dimensions, data, texture, } } pub fn get_linear(&self, x: u32, y: u32) -> f32 { self.data[(x + y * self.dimensions.x) as usize] } } struct AlbedoTexture { dimensions: glam::UVec2, rgba8: Vec, texture: GpuTexture2D, } impl AlbedoTexture { pub fn spiral(re_ctx: &re_renderer::RenderContext, dimensions: glam::UVec2) -> Self { let size = (dimensions.x * dimensions.y) as usize; let mut rgba8 = vec![0; size * 4]; spiral(dimensions).for_each(|(texcoords, d)| { let idx = ((texcoords.x + texcoords.y * dimensions.x) * 4) as usize; rgba8[idx..idx + 4].copy_from_slice(re_renderer::colormap_turbo_srgba(d).as_slice()); }); let label = format!("albedo texture spiral {dimensions}"); let texture = re_ctx .texture_manager_2d .get_or_create( hash(&label), re_ctx, ImageDataDesc { label: label.into(), data: bytemuck::cast_slice(&rgba8).into(), format: wgpu::TextureFormat::Rgba8UnormSrgb.into(), width_height: dimensions.to_array(), alpha_channel_usage: re_renderer::AlphaChannelUsage::Opaque, }, ) .expect("Failed to create albedo texture."); Self { dimensions, rgba8, texture, } } #[expect(dead_code)] pub fn get(&self, x: u32, y: u32) -> [u8; 4] { let p = &self.rgba8[(x + y * self.dimensions.x) as usize * 4..]; [p[0], p[1], p[2], p[3]] } }