Merge pull request #104310 from mihe/jolt/back-face-collision

Fix `ConcavePolygonShape3D` always enabling `backface_collision` when using Jolt Physics

Author: akien-mga

modules/jolt_physics/shapes/jolt_custom_double_sided_shape.cpp

@@ -49,7 +49,9 @@ void collide_double_sided_vs_shape(const JPH::Shape *p_shape1, const JPH::Shape
 	const JoltCustomDoubleSidedShape *shape1 = static_cast<const JoltCustomDoubleSidedShape *>(p_shape1);
 
 	JPH::CollideShapeSettings new_collide_shape_settings = p_collide_shape_settings;
-	new_collide_shape_settings.mBackFaceMode = JPH::EBackFaceMode::CollideWithBackFaces;
+	if (shape1->should_collide_with_back_faces()) {
+		new_collide_shape_settings.mBackFaceMode = JPH::EBackFaceMode::CollideWithBackFaces;
+	}
 
 	JPH::CollisionDispatch::sCollideShapeVsShape(shape1->GetInnerShape(), p_shape2, p_scale1, p_scale2, p_center_of_mass_transform1, p_center_of_mass_transform2, p_sub_shape_id_creator1, p_sub_shape_id_creator2, new_collide_shape_settings, p_collector, p_shape_filter);
 }
@@ -60,7 +62,9 @@ void collide_shape_vs_double_sided(const JPH::Shape *p_shape1, const JPH::Shape
 	const JoltCustomDoubleSidedShape *shape2 = static_cast<const JoltCustomDoubleSidedShape *>(p_shape2);
 
 	JPH::CollideShapeSettings new_collide_shape_settings = p_collide_shape_settings;
-	new_collide_shape_settings.mBackFaceMode = JPH::EBackFaceMode::CollideWithBackFaces;
+	if (shape2->should_collide_with_back_faces()) {
+		new_collide_shape_settings.mBackFaceMode = JPH::EBackFaceMode::CollideWithBackFaces;
+	}
 
 	JPH::CollisionDispatch::sCollideShapeVsShape(p_shape1, shape2->GetInnerShape(), p_scale1, p_scale2, p_center_of_mass_transform1, p_center_of_mass_transform2, p_sub_shape_id_creator1, p_sub_shape_id_creator2, new_collide_shape_settings, p_collector, p_shape_filter);
 }
@@ -71,7 +75,9 @@ void cast_shape_vs_double_sided(const JPH::ShapeCast &p_shape_cast, const JPH::S
 	const JoltCustomDoubleSidedShape *shape = static_cast<const JoltCustomDoubleSidedShape *>(p_shape);
 
 	JPH::ShapeCastSettings new_shape_cast_settings = p_shape_cast_settings;
-	new_shape_cast_settings.mBackFaceModeTriangles = JPH::EBackFaceMode::CollideWithBackFaces;
+	if (shape->should_collide_with_back_faces()) {
+		new_shape_cast_settings.mBackFaceModeTriangles = JPH::EBackFaceMode::CollideWithBackFaces;
+	}
 
 	JPH::CollisionDispatch::sCastShapeVsShapeLocalSpace(p_shape_cast, new_shape_cast_settings, shape->GetInnerShape(), p_scale, p_shape_filter, p_center_of_mass_transform2, p_sub_shape_id_creator1, p_sub_shape_id_creator2, p_collector);
 }
@@ -104,8 +110,7 @@ void JoltCustomDoubleSidedShape::register_type() {
 
 void JoltCustomDoubleSidedShape::CastRay(const JPH::RayCast &p_ray, const JPH::RayCastSettings &p_ray_cast_settings, const JPH::SubShapeIDCreator &p_sub_shape_id_creator, JPH::CastRayCollector &p_collector, const JPH::ShapeFilter &p_shape_filter) const {
 	JPH::RayCastSettings new_ray_cast_settings = p_ray_cast_settings;
-
-	if (!back_face_collision) {
+	if (!should_collide_with_back_faces()) {
 		new_ray_cast_settings.SetBackFaceMode(JPH::EBackFaceMode::IgnoreBackFaces);
 	}
 

modules/jolt_physics/shapes/jolt_custom_double_sided_shape.h

@@ -68,4 +68,6 @@ class JoltCustomDoubleSidedShape final : public JoltCustomDecoratedShape {
 			JoltCustomDecoratedShape(JoltCustomShapeSubType::DOUBLE_SIDED, p_inner_shape), back_face_collision(p_back_face_collision) {}
 
 	virtual void CastRay(const JPH::RayCast &p_ray, const JPH::RayCastSettings &p_ray_cast_settings, const JPH::SubShapeIDCreator &p_sub_shape_id_creator, JPH::CastRayCollector &p_collector, const JPH::ShapeFilter &p_shape_filter = JPH::ShapeFilter()) const override;
+
+	bool should_collide_with_back_faces() const { return back_face_collision; }
 };