/* * Software License Agreement (BSD License) * * Point Cloud Library (PCL) - www.pointclouds.org * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials provided * with the distribution. * * Neither the name of the copyright holder(s) nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * Author : Christian Potthast * Email : potthast@usc.edu * */ #ifndef PCL_CRF_SEGMENTATION_HPP_ #define PCL_CRF_SEGMENTATION_HPP_ #include // for VoxelGridLabel #include #include #include #include ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template pcl::CrfSegmentation::CrfSegmentation () : voxel_grid_ (), input_cloud_ (new pcl::PointCloud), normal_cloud_ (new pcl::PointCloud), filtered_cloud_ (new pcl::PointCloud), filtered_anno_ (new pcl::PointCloud), filtered_normal_ (new pcl::PointCloud), voxel_grid_leaf_size_ (Eigen::Vector4f (0.001f, 0.001f, 0.001f, 0.0f)) { } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template pcl::CrfSegmentation::~CrfSegmentation () { } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setInputCloud (typename pcl::PointCloud::Ptr input_cloud) { input_cloud_ = input_cloud; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setAnnotatedCloud (typename pcl::PointCloud::Ptr anno_cloud) { anno_cloud_ = anno_cloud; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setNormalCloud (typename pcl::PointCloud::Ptr normal_cloud) { normal_cloud_ = normal_cloud; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setVoxelGridLeafSize (const float x, const float y, const float z) { voxel_grid_leaf_size_.x () = x; voxel_grid_leaf_size_.y () = y; voxel_grid_leaf_size_.z () = z; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setSmoothnessKernelParameters (const float sx, const float sy, const float sz, const float w) { smoothness_kernel_param_[0] = sx; smoothness_kernel_param_[1] = sy; smoothness_kernel_param_[2] = sz; smoothness_kernel_param_[3] = w; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setAppearanceKernelParameters (float sx, float sy, float sz, float sr, float sg, float sb, float w) { appearance_kernel_param_[0] = sx; appearance_kernel_param_[1] = sy; appearance_kernel_param_[2] = sz; appearance_kernel_param_[3] = sr; appearance_kernel_param_[4] = sg; appearance_kernel_param_[5] = sb; appearance_kernel_param_[6] = w; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::setSurfaceKernelParameters (float sx, float sy, float sz, float snx, float sny, float snz, float w) { surface_kernel_param_[0] = sx; surface_kernel_param_[1] = sy; surface_kernel_param_[2] = sz; surface_kernel_param_[3] = snx; surface_kernel_param_[4] = sny; surface_kernel_param_[5] = snz; surface_kernel_param_[6] = w; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::createVoxelGrid () { // Filter the input cloud // Set the voxel grid input cloud voxel_grid_.setInputCloud (input_cloud_); // Set the voxel grid leaf size voxel_grid_.setLeafSize (voxel_grid_leaf_size_.x (), voxel_grid_leaf_size_.y (), voxel_grid_leaf_size_.z () ); // Only downsample XYZ (if this is set to false, color values set to 0) voxel_grid_.setDownsampleAllData (true); // Save leaf information //voxel_grid_.setSaveLeafLayout (true); // apply the filter voxel_grid_.filter (*filtered_cloud_); // Filter the annotated cloud if (!anno_cloud_->points.empty ()) { pcl::VoxelGridLabel vg; vg.setInputCloud (anno_cloud_); // Set the voxel grid leaf size vg.setLeafSize (voxel_grid_leaf_size_.x (), voxel_grid_leaf_size_.y (), voxel_grid_leaf_size_.z () ); // Only downsample XYZ vg.setDownsampleAllData (true); // Save leaf information //vg.setSaveLeafLayout (false); // apply the filter vg.filter (*filtered_anno_); } // Filter the annotated cloud if (!normal_cloud_->points.empty ()) { pcl::VoxelGrid vg; vg.setInputCloud (normal_cloud_); // Set the voxel grid leaf size vg.setLeafSize (voxel_grid_leaf_size_.x (), voxel_grid_leaf_size_.y (), voxel_grid_leaf_size_.z () ); // Only downsample XYZ vg.setDownsampleAllData (true); // Save leaf information //vg.setSaveLeafLayout (false); // apply the filter vg.filter (*filtered_normal_); } } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::createDataVectorFromVoxelGrid () { // get the dimension of the voxel grid //Eigen::Vector3i min_b, max_b; //min_b = voxel_grid_.getMinBoxCoordinates (); //max_b = voxel_grid_.getMaxBoxCoordinates (); //std::cout << "min_b: " << min_b.x () << " " << min_b.y () << " " << min_b.z () << std::endl; //std::cout << "max_b: " << max_b.x () << " " << max_b.y () << " " << max_b.z () << std::endl; // compute the voxel grid dimensions //dim_.x () = std::abs (max_b.x () - min_b.x ()); //dim_.y () = std::abs (max_b.y () - min_b.y ()); //dim_.z () = std::abs (max_b.z () - min_b.z ()); //std::cout << dim_.x () * dim_.y () * dim_.z () << std::endl; // reserve the space for the data vector //data_.reserve (dim_.x () * dim_.y () * dim_.z ()); /* std::vector > data; std::vector > color; // fill the data vector for (int kk = min_b.z (), k = 0; kk <= max_b.z (); kk++, k++) { for (int jj = min_b.y (), j = 0; jj <= max_b.y (); jj++, j++) { for (int ii = min_b.x (), i = 0; ii <= max_b.x (); ii++, i++) { Eigen::Vector3i ijk (ii, jj, kk); int index = voxel_grid_.getCentroidIndexAt (ijk); if (index != -1) { data_.push_back (Eigen::Vector3i (i, j, k)); color_.push_back ((*input_cloud_)[index].getRGBVector3i ()); } } } } */ /* // get the size of the input fields std::vector< pcl::PCLPointField > fields; pcl::getFields (*input_cloud_, fields); for (int i = 0; i < fields.size (); i++) std::cout << fields[i] << std::endl; */ // reserve space for the data vector data_.resize (filtered_cloud_->size ()); std::vector< pcl::PCLPointField > fields; // check if we have color data bool color_data = false; int rgba_index = -1; rgba_index = pcl::getFieldIndex ("rgb", fields); if (rgba_index == -1) rgba_index = pcl::getFieldIndex ("rgba", fields); if (rgba_index >= 0) { color_data = true; color_.resize (filtered_cloud_->size ()); } /* // check if we have normal data bool normal_data = false; int normal_index = -1; rgba_index = pcl::getFieldIndex ("normal_x", fields); if (rgba_index >= 0) { normal_data = true; normal_.resize (filtered_cloud_->size ()); } */ // fill the data vector for (std::size_t i = 0; i < filtered_cloud_->size (); i++) { Eigen::Vector3f p ((*filtered_anno_)[i].x, (*filtered_anno_)[i].y, (*filtered_anno_)[i].z); Eigen::Vector3i c = voxel_grid_.getGridCoordinates (p.x (), p.y (), p.y ()); data_[i] = c; if (color_data) { std::uint32_t rgb = *reinterpret_cast(&(*filtered_cloud_)[i].rgba); std::uint8_t r = (rgb >> 16) & 0x0000ff; std::uint8_t g = (rgb >> 8) & 0x0000ff; std::uint8_t b = (rgb) & 0x0000ff; color_[i] = Eigen::Vector3i (r, g, b); } /* if (normal_data) { float n_x = (*filtered_cloud_)[i].normal_x; float n_y = (*filtered_cloud_)[i].normal_y; float n_z = (*filtered_cloud_)[i].normal_z; normal_[i] = Eigen::Vector3f (n_x, n_y, n_z); } */ } normal_.resize (filtered_normal_->size ()); for (std::size_t i = 0; i < filtered_normal_->size (); i++) { float n_x = (*filtered_normal_)[i].normal_x; float n_y = (*filtered_normal_)[i].normal_y; float n_z = (*filtered_normal_)[i].normal_z; normal_[i] = Eigen::Vector3f (n_x, n_y, n_z); } } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::createUnaryPotentials (std::vector &unary, std::vector &labels, unsigned int n_labels) { /* initialize random seed: */ srand ( static_cast (time (nullptr)) ); //srand ( time (NULL) ); // Certainty that the groundtruth is correct const float GT_PROB = 0.9f; const float u_energy = -std::log ( 1.0f / static_cast (n_labels) ); const float n_energy = -std::log ( (1.0f - GT_PROB) / static_cast(n_labels - 1) ); const float p_energy = -std::log ( GT_PROB ); for (std::size_t k = 0; k < filtered_anno_->size (); k++) { int label = (*filtered_anno_)[k].label; if (labels.empty () && label > 0) labels.push_back (label); // add color to the color vector if not added yet for (int c_idx = 0; c_idx < static_cast (labels.size ()) ; c_idx++) { if (labels[c_idx] == label) break; if (c_idx == static_cast(labels.size () -1) && label > 0) { if (labels.size () < n_labels) labels.push_back (label); else label = 0; } } // set the engeries for the labels std::size_t u_idx = k * n_labels; if (label > 0) { for (std::size_t i = 0; i < n_labels; i++) unary[u_idx + i] = n_energy; unary[u_idx + labels.size ()] = p_energy; if (label == 1) { const float PROB = 0.2f; const float n_energy2 = -std::log ( (1.0f - PROB) / static_cast(n_labels - 1) ); const float p_energy2 = -std::log ( PROB ); for (std::size_t i = 0; i < n_labels; i++) unary[u_idx + i] = n_energy2; unary[u_idx + labels.size ()] = p_energy2; } } else { for (std::size_t i = 0; i < n_labels; i++) unary[u_idx + i] = u_energy; } } } ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// template void pcl::CrfSegmentation::segmentPoints (pcl::PointCloud &output) { // create the voxel grid createVoxelGrid (); std::cout << "create Voxel Grid - DONE" << std::endl; // create the data Vector createDataVectorFromVoxelGrid (); std::cout << "create Data Vector from Voxel Grid - DONE" << std::endl; // number of labels const int n_labels = 10; // number of data points int N = static_cast (data_.size ()); // create unary potentials std::vector labels; std::vector unary; if (!anno_cloud_->points.empty ()) { unary.resize (N * n_labels); createUnaryPotentials (unary, labels, n_labels); std::cout << "labels size: " << labels.size () << std::endl; for (const int &label : labels) { std::cout << label << std::endl; } } std::cout << "create unary potentials - DONE" << std::endl; /* pcl::PointCloud tmp_cloud_OLD; tmp_cloud_OLD = *filtered_anno_; // Setup the CRF model DenseCRF2D crfOLD(N, 1, n_labels); float * unaryORI = new float[N*n_labels]; for (int i = 0; i < N*n_labels; i++) unaryORI[i] = unary[i]; crfOLD.setUnaryEnergy( unaryORI ); float * pos = new float[N*3]; for (int i = 0; i < N; i++) { pos[i * 3] = data_[i].x (); pos[i * 3 +1] = data_[i].y (); pos[i * 3 +2] = data_[i].z (); } crfOLD.addPairwiseGaussian( pos, 3, 3, 3, 2.0 ); float * col = new float[N*3]; for (int i = 0; i < N; i++) { col[i * 3] = color_[i].x (); col[i * 3 +1] = color_[i].y (); col[i * 3 +2] = color_[i].z (); } crfOLD.addPairwiseBilateral(pos, col, 20, 20, 20, 10, 10, 10, 1.3 ); short * map = new short[N]; crfOLD.map(10, map); for (std::size_t i = 0; i < N; i++) { tmp_cloud_OLD[i].label = map[i]; } */ //float * resultOLD = new float[N*n_labels]; //crfOLD.inference (10, resultOLD); //std::vector baryOLD; //crfOLD.getBarycentric (0, baryOLD); //std::vector featuresOLD; //crfOLD.getFeature (1, featuresOLD); /* for(int i = 0; i < 25; i++) { std::cout << baryOLD[i] << std::endl; } */ // create the output cloud //output = *filtered_anno_; // ----------------------------------// // -------- -------------------// // create dense CRF DenseCrf crf (N, n_labels); // set the unary potentials crf.setUnaryEnergy (unary); // set the data vector crf.setDataVector (data_); // set the color vector crf.setColorVector (color_); std::cout << "create dense CRF - DONE" << std::endl; // add the smoothness kernel crf.addPairwiseGaussian (smoothness_kernel_param_[0], smoothness_kernel_param_[1], smoothness_kernel_param_[2], smoothness_kernel_param_[3]); std::cout << "add smoothness kernel - DONE" << std::endl; // add the appearance kernel crf.addPairwiseBilateral (appearance_kernel_param_[0], appearance_kernel_param_[1], appearance_kernel_param_[2], appearance_kernel_param_[3], appearance_kernel_param_[4], appearance_kernel_param_[5], appearance_kernel_param_[6]); std::cout << "add appearance kernel - DONE" << std::endl; crf.addPairwiseNormals (data_, normal_, surface_kernel_param_[0], surface_kernel_param_[1], surface_kernel_param_[2], surface_kernel_param_[3], surface_kernel_param_[4], surface_kernel_param_[5], surface_kernel_param_[6]); std::cout << "add surface kernel - DONE" << std::endl; // map inference std::vector r (N); crf.mapInference (n_iterations_, r); //std::vector result (N*n_labels); //crf.inference (n_iterations_, result); //std::vector bary; //crf.getBarycentric (0, bary); //std::vector features; //crf.getFeatures (1, features); std::cout << "Map inference - DONE" << std::endl; // create the output cloud output = *filtered_anno_; for (int i = 0; i < N; i++) { output[i].label = labels[r[i]]; } /* pcl::PointCloud tmp_cloud; tmp_cloud = *filtered_anno_; bool c = true; for (std::size_t i = 0; i < tmp_cloud.size (); i++) { if (tmp_cloud[i].label != tmp_cloud_OLD[i].label) { std::cout << "idx: " << i << " = " <; #endif // PCL_CRF_SEGMENTATION_HPP_