Blender  V3.3
intern/reconstruction.cc
Go to the documentation of this file.
1 /* SPDX-License-Identifier: GPL-2.0-or-later
2  * Copyright 2011 Blender Foundation. All rights reserved. */
3 
6 #include "intern/tracks.h"
7 #include "intern/utildefines.h"
8 
17 
23 using libmv::Marker;
25 
31 using libmv::Tracks;
32 
35 
36  /* Used for per-track average error calculation after reconstruction */
39 
40  double error;
41  bool is_valid;
42 };
43 
44 namespace {
45 
46 class ReconstructUpdateCallback : public ProgressUpdateCallback {
47  public:
48  ReconstructUpdateCallback(
49  reconstruct_progress_update_cb progress_update_callback,
50  void* callback_customdata) {
51  progress_update_callback_ = progress_update_callback;
52  callback_customdata_ = callback_customdata;
53  }
54 
55  void invoke(double progress, const char* message) {
56  if (progress_update_callback_) {
57  progress_update_callback_(callback_customdata_, progress, message);
58  }
59  }
60 
61  protected:
62  reconstruct_progress_update_cb progress_update_callback_;
63  void* callback_customdata_;
64 };
65 
66 void libmv_solveRefineIntrinsics(
67  const Tracks& tracks,
68  const int refine_intrinsics,
69  const int bundle_constraints,
70  reconstruct_progress_update_cb progress_update_callback,
71  void* callback_customdata,
73  CameraIntrinsics* intrinsics) {
74  /* only a few combinations are supported but trust the caller/ */
75  int bundle_intrinsics = 0;
76 
77  if (refine_intrinsics & LIBMV_REFINE_FOCAL_LENGTH) {
78  bundle_intrinsics |= libmv::BUNDLE_FOCAL_LENGTH;
79  }
80  if (refine_intrinsics & LIBMV_REFINE_PRINCIPAL_POINT) {
81  bundle_intrinsics |= libmv::BUNDLE_PRINCIPAL_POINT;
82  }
83 
84 #define SET_DISTORTION_FLAG_CHECKED(type, coefficient) \
85  do { \
86  if (refine_intrinsics & LIBMV_REFINE_##type##_DISTORTION_##coefficient) { \
87  bundle_intrinsics |= libmv::BUNDLE_##type##_##coefficient; \
88  } \
89  } while (0)
90 
91  SET_DISTORTION_FLAG_CHECKED(RADIAL, K1);
92  SET_DISTORTION_FLAG_CHECKED(RADIAL, K2);
93  SET_DISTORTION_FLAG_CHECKED(RADIAL, K3);
94  SET_DISTORTION_FLAG_CHECKED(RADIAL, K4);
95 
98 
99 #undef SET_DISTORTION_FLAG_CHECKED
100 
101  progress_update_callback(callback_customdata, 1.0, "Refining solution");
102 
104  bundle_intrinsics,
105  bundle_constraints,
107  intrinsics);
108 }
109 
110 void finishReconstruction(
111  const Tracks& tracks,
112  const CameraIntrinsics& camera_intrinsics,
113  libmv_Reconstruction* libmv_reconstruction,
114  reconstruct_progress_update_cb progress_update_callback,
115  void* callback_customdata) {
117  libmv_reconstruction->reconstruction;
118 
119  /* Reprojection error calculation. */
120  progress_update_callback(callback_customdata, 1.0, "Finishing solution");
121  libmv_reconstruction->tracks = tracks;
122  libmv_reconstruction->error =
123  EuclideanReprojectionError(tracks, reconstruction, camera_intrinsics);
124 }
125 
126 bool selectTwoKeyframesBasedOnGRICAndVariance(
127  Tracks& tracks,
128  Tracks& normalized_tracks,
129  CameraIntrinsics& camera_intrinsics,
130  int& keyframe1,
131  int& keyframe2) {
132  libmv::vector<int> keyframes;
133 
134  /* Get list of all keyframe candidates first. */
136  normalized_tracks, camera_intrinsics, keyframes);
137 
138  if (keyframes.size() < 2) {
139  LG << "Not enough keyframes detected by GRIC";
140  return false;
141  } else if (keyframes.size() == 2) {
142  keyframe1 = keyframes[0];
143  keyframe2 = keyframes[1];
144  return true;
145  }
146 
147  /* Now choose two keyframes with minimal reprojection error after initial
148  * reconstruction choose keyframes with the least reprojection error after
149  * solving from two candidate keyframes.
150  *
151  * In fact, currently libmv returns single pair only, so this code will
152  * not actually run. But in the future this could change, so let's stay
153  * prepared.
154  */
155  int previous_keyframe = keyframes[0];
156  double best_error = std::numeric_limits<double>::max();
157  for (int i = 1; i < keyframes.size(); i++) {
159  int current_keyframe = keyframes[i];
160  libmv::vector<Marker> keyframe_markers =
161  normalized_tracks.MarkersForTracksInBothImages(previous_keyframe,
162  current_keyframe);
163 
164  Tracks keyframe_tracks(keyframe_markers);
165 
166  /* get a solution from two keyframes only */
167  EuclideanReconstructTwoFrames(keyframe_markers, &reconstruction);
168  EuclideanBundle(keyframe_tracks, &reconstruction);
170 
171  double current_error =
172  EuclideanReprojectionError(tracks, reconstruction, camera_intrinsics);
173 
174  LG << "Error between " << previous_keyframe << " and " << current_keyframe
175  << ": " << current_error;
176 
177  if (current_error < best_error) {
178  best_error = current_error;
179  keyframe1 = previous_keyframe;
180  keyframe2 = current_keyframe;
181  }
182 
183  previous_keyframe = current_keyframe;
184  }
185 
186  return true;
187 }
188 
189 Marker libmv_projectMarker(const EuclideanPoint& point,
190  const EuclideanCamera& camera,
191  const CameraIntrinsics& intrinsics) {
192  libmv::Vec3 projected = camera.R * point.X + camera.t;
193  projected /= projected(2);
194 
195  libmv::Marker reprojected_marker;
196  intrinsics.ApplyIntrinsics(
197  projected(0), projected(1), &reprojected_marker.x, &reprojected_marker.y);
198 
199  reprojected_marker.image = camera.image;
200  reprojected_marker.track = point.track;
201  return reprojected_marker;
202 }
203 
204 void libmv_getNormalizedTracks(const Tracks& tracks,
205  const CameraIntrinsics& camera_intrinsics,
206  Tracks* normalized_tracks) {
207  libmv::vector<Marker> markers = tracks.AllMarkers();
208  for (int i = 0; i < markers.size(); ++i) {
209  Marker& marker = markers[i];
210  camera_intrinsics.InvertIntrinsics(
211  marker.x, marker.y, &marker.x, &marker.y);
212  normalized_tracks->Insert(
213  marker.image, marker.track, marker.x, marker.y, marker.weight);
214  }
215 }
216 
217 } // namespace
218 
220  const libmv_Tracks* libmv_tracks,
221  const libmv_CameraIntrinsicsOptions* libmv_camera_intrinsics_options,
222  libmv_ReconstructionOptions* libmv_reconstruction_options,
223  reconstruct_progress_update_cb progress_update_callback,
224  void* callback_customdata) {
225  libmv_Reconstruction* libmv_reconstruction =
227 
228  Tracks& tracks = *((Tracks*)libmv_tracks);
230  libmv_reconstruction->reconstruction;
231 
232  ReconstructUpdateCallback update_callback =
233  ReconstructUpdateCallback(progress_update_callback, callback_customdata);
234 
235  /* Retrieve reconstruction options from C-API to libmv API. */
236  CameraIntrinsics* camera_intrinsics;
237  camera_intrinsics = libmv_reconstruction->intrinsics =
238  libmv_cameraIntrinsicsCreateFromOptions(libmv_camera_intrinsics_options);
239 
240  /* Invert the camera intrinsics/ */
241  Tracks normalized_tracks;
242  libmv_getNormalizedTracks(tracks, *camera_intrinsics, &normalized_tracks);
243 
244  /* keyframe selection. */
245  int keyframe1 = libmv_reconstruction_options->keyframe1,
246  keyframe2 = libmv_reconstruction_options->keyframe2;
247 
248  if (libmv_reconstruction_options->select_keyframes) {
249  LG << "Using automatic keyframe selection";
250 
251  update_callback.invoke(0, "Selecting keyframes");
252 
253  if (selectTwoKeyframesBasedOnGRICAndVariance(tracks,
254  normalized_tracks,
255  *camera_intrinsics,
256  keyframe1,
257  keyframe2)) {
258  /* so keyframes in the interface would be updated */
259  libmv_reconstruction_options->keyframe1 = keyframe1;
260  libmv_reconstruction_options->keyframe2 = keyframe2;
261  }
262  }
263 
264  /* Actual reconstruction. */
265  LG << "frames to init from: " << keyframe1 << " " << keyframe2;
266 
267  libmv::vector<Marker> keyframe_markers =
268  normalized_tracks.MarkersForTracksInBothImages(keyframe1, keyframe2);
269 
270  LG << "number of markers for init: " << keyframe_markers.size();
271 
272  if (keyframe_markers.size() < 16) {
273  LG << "No enough markers to initialize from";
274  libmv_reconstruction->is_valid = false;
275  return libmv_reconstruction;
276  }
277 
278  update_callback.invoke(0, "Initial reconstruction");
279 
280  if (!EuclideanReconstructTwoFrames(keyframe_markers, &reconstruction)) {
281  LG << "Failed to initialize reconstruction";
282  libmv_reconstruction->is_valid = false;
283  return libmv_reconstruction;
284  }
285 
286  EuclideanBundle(normalized_tracks, &reconstruction);
288  normalized_tracks, &reconstruction, &update_callback);
289 
290  /* Refinement. */
291  if (libmv_reconstruction_options->refine_intrinsics) {
292  libmv_solveRefineIntrinsics(tracks,
293  libmv_reconstruction_options->refine_intrinsics,
295  progress_update_callback,
296  callback_customdata,
298  camera_intrinsics);
299  }
300 
301  /* Set reconstruction scale to unity. */
303 
304  /* Finish reconstruction. */
305  finishReconstruction(tracks,
306  *camera_intrinsics,
307  libmv_reconstruction,
308  progress_update_callback,
309  callback_customdata);
310 
311  libmv_reconstruction->is_valid = true;
312  return (libmv_Reconstruction*)libmv_reconstruction;
313 }
314 
316  const libmv_Tracks* libmv_tracks,
317  const libmv_CameraIntrinsicsOptions* libmv_camera_intrinsics_options,
318  const libmv_ReconstructionOptions* libmv_reconstruction_options,
319  reconstruct_progress_update_cb progress_update_callback,
320  void* callback_customdata) {
321  libmv_Reconstruction* libmv_reconstruction =
323 
324  Tracks& tracks = *((Tracks*)libmv_tracks);
326  libmv_reconstruction->reconstruction;
327 
328  ReconstructUpdateCallback update_callback =
329  ReconstructUpdateCallback(progress_update_callback, callback_customdata);
330 
331  /* Retrieve reconstruction options from C-API to libmv API. */
332  CameraIntrinsics* camera_intrinsics;
333  camera_intrinsics = libmv_reconstruction->intrinsics =
334  libmv_cameraIntrinsicsCreateFromOptions(libmv_camera_intrinsics_options);
335 
336  /* Invert the camera intrinsics. */
337  Tracks normalized_tracks;
338  libmv_getNormalizedTracks(tracks, *camera_intrinsics, &normalized_tracks);
339 
340  /* Actual reconstruction. */
341  ModalSolver(normalized_tracks, &reconstruction, &update_callback);
342 
343  PolynomialCameraIntrinsics empty_intrinsics;
344  EuclideanBundleCommonIntrinsics(normalized_tracks,
348  &empty_intrinsics);
349 
350  /* Refinement. */
351  if (libmv_reconstruction_options->refine_intrinsics) {
352  libmv_solveRefineIntrinsics(tracks,
353  libmv_reconstruction_options->refine_intrinsics,
355  progress_update_callback,
356  callback_customdata,
358  camera_intrinsics);
359  }
360 
361  /* Finish reconstruction. */
362  finishReconstruction(tracks,
363  *camera_intrinsics,
364  libmv_reconstruction,
365  progress_update_callback,
366  callback_customdata);
367 
368  libmv_reconstruction->is_valid = true;
369  return (libmv_Reconstruction*)libmv_reconstruction;
370 }
371 
373  return libmv_reconstruction->is_valid;
374 }
375 
377  LIBMV_OBJECT_DELETE(libmv_reconstruction->intrinsics, CameraIntrinsics);
378  LIBMV_OBJECT_DELETE(libmv_reconstruction, libmv_Reconstruction);
379 }
380 
382  const libmv_Reconstruction* libmv_reconstruction,
383  int track,
384  double pos[3]) {
386  &libmv_reconstruction->reconstruction;
387  const EuclideanPoint* point = reconstruction->PointForTrack(track);
388  if (point) {
389  pos[0] = point->X[0];
390  pos[1] = point->X[2];
391  pos[2] = point->X[1];
392  return 1;
393  }
394  return 0;
395 }
396 
398  const libmv_Reconstruction* libmv_reconstruction, int track) {
400  &libmv_reconstruction->reconstruction;
401  const CameraIntrinsics* intrinsics = libmv_reconstruction->intrinsics;
403  libmv_reconstruction->tracks.MarkersForTrack(track);
404 
405  int num_reprojected = 0;
406  double total_error = 0.0;
407 
408  for (int i = 0; i < markers.size(); ++i) {
409  double weight = markers[i].weight;
410  const EuclideanCamera* camera =
411  reconstruction->CameraForImage(markers[i].image);
412  const EuclideanPoint* point =
413  reconstruction->PointForTrack(markers[i].track);
414 
415  if (!camera || !point || weight == 0.0) {
416  continue;
417  }
418 
419  num_reprojected++;
420 
421  Marker reprojected_marker =
422  libmv_projectMarker(*point, *camera, *intrinsics);
423  double ex = (reprojected_marker.x - markers[i].x) * weight;
424  double ey = (reprojected_marker.y - markers[i].y) * weight;
425 
426  total_error += sqrt(ex * ex + ey * ey);
427  }
428 
429  return total_error / num_reprojected;
430 }
431 
433  const libmv_Reconstruction* libmv_reconstruction, int image) {
435  &libmv_reconstruction->reconstruction;
436  const CameraIntrinsics* intrinsics = libmv_reconstruction->intrinsics;
438  libmv_reconstruction->tracks.MarkersInImage(image);
439  const EuclideanCamera* camera = reconstruction->CameraForImage(image);
440  int num_reprojected = 0;
441  double total_error = 0.0;
442 
443  if (!camera) {
444  return 0.0;
445  }
446 
447  for (int i = 0; i < markers.size(); ++i) {
448  const EuclideanPoint* point =
449  reconstruction->PointForTrack(markers[i].track);
450 
451  if (!point) {
452  continue;
453  }
454 
455  num_reprojected++;
456 
457  Marker reprojected_marker =
458  libmv_projectMarker(*point, *camera, *intrinsics);
459  double ex = (reprojected_marker.x - markers[i].x) * markers[i].weight;
460  double ey = (reprojected_marker.y - markers[i].y) * markers[i].weight;
461 
462  total_error += sqrt(ex * ex + ey * ey);
463  }
464 
465  return total_error / num_reprojected;
466 }
467 
469  const libmv_Reconstruction* libmv_reconstruction,
470  int image,
471  double mat[4][4]) {
473  &libmv_reconstruction->reconstruction;
474  const EuclideanCamera* camera = reconstruction->CameraForImage(image);
475 
476  if (camera) {
477  for (int j = 0; j < 3; ++j) {
478  for (int k = 0; k < 3; ++k) {
479  int l = k;
480 
481  if (k == 1) {
482  l = 2;
483  } else if (k == 2) {
484  l = 1;
485  }
486 
487  if (j == 2) {
488  mat[j][l] = -camera->R(j, k);
489  } else {
490  mat[j][l] = camera->R(j, k);
491  }
492  }
493  mat[j][3] = 0.0;
494  }
495 
496  libmv::Vec3 optical_center = -camera->R.transpose() * camera->t;
497 
498  mat[3][0] = optical_center(0);
499  mat[3][1] = optical_center(2);
500  mat[3][2] = optical_center(1);
501 
502  mat[3][3] = 1.0;
503 
504  return 1;
505  }
506 
507  return 0;
508 }
509 
511  const libmv_Reconstruction* libmv_reconstruction) {
512  return libmv_reconstruction->error;
513 }
514 
516  libmv_Reconstruction* libmv_reconstruction) {
517  return (libmv_CameraIntrinsics*)libmv_reconstruction->intrinsics;
518 }
sqrt(x)+1/max(0
in reality light always falls off quadratically Particle Retrieve the data of the particle that spawned the object for example to give variation to multiple instances of an object Point Retrieve information about points in a point cloud Retrieve the edges of an object as it appears to Cycles topology will always appear triangulated Convert a blackbody temperature to an RGB value Normal Generate a perturbed normal from an RGB normal map image Typically used for faking highly detailed surfaces Generate an OSL shader from a file or text data block Image Sample an image file as a texture Sky Generate a procedural sky texture Noise Generate fractal Perlin noise Wave Generate procedural bands or rings with noise Voronoi Generate Worley noise based on the distance to random points Typically used to generate textures such as or biological cells Brick Generate a procedural texture producing bricks Texture Retrieve multiple types of texture coordinates nTypically used as inputs for texture nodes Vector Convert a point
in reality light always falls off quadratically Particle Retrieve the data of the particle that spawned the object for example to give variation to multiple instances of an object Point Retrieve information about points in a point cloud Retrieve the edges of an object as it appears to Cycles topology will always appear triangulated Convert a blackbody temperature to an RGB value Normal Generate a perturbed normal from an RGB normal map image Typically used for faking highly detailed surfaces Generate an OSL shader from a file or text data block Image Sample an image file as a texture Sky Generate a procedural sky texture Noise Generate fractal Perlin noise Wave Generate procedural bands or rings with noise Voronoi Generate Worley noise based on the distance to random points Typically used to generate textures such as or biological cells Brick Generate a procedural texture producing bricks Texture Retrieve multiple types of texture coordinates nTypically used as inputs for texture nodes Vector Convert a or normal between camera
ATTR_WARN_UNUSED_RESULT const BMLoop * l
virtual void ApplyIntrinsics(double normalized_x, double normalized_y, double *image_x, double *image_y) const =0
virtual void InvertIntrinsics(double image_x, double image_y, double *normalized_x, double *normalized_y) const =0
virtual void invoke(double progress, const char *message)=0
vector< Marker > MarkersInImage(int image) const
Returns all the markers visible in image.
vector< Marker > MarkersForTracksInBothImages(int image1, int image2) const
void Insert(int image, int track, double x, double y, double weight=1.0)
vector< Marker > MarkersForTrack(int track) const
Returns all the markers belonging to a track.
depth_tx normal_tx diffuse_light_tx specular_light_tx volume_light_tx environment_tx ambient_occlusion_tx aov_value_tx in_weight_img image(1, GPU_R32F, Qualifier::WRITE, ImageType::FLOAT_2D_ARRAY, "out_weight_img") .image(3
uint pos
const vector< Marker > & markers
CameraIntrinsics * libmv_cameraIntrinsicsCreateFromOptions(const libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
struct libmv_CameraIntrinsics libmv_CameraIntrinsics
libmv_Reconstruction * libmv_solveReconstruction(const libmv_Tracks *libmv_tracks, const libmv_CameraIntrinsicsOptions *libmv_camera_intrinsics_options, libmv_ReconstructionOptions *libmv_reconstruction_options, reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
double libmv_reprojectionErrorForImage(const libmv_Reconstruction *libmv_reconstruction, int image)
libmv_CameraIntrinsics * libmv_reconstructionExtractIntrinsics(libmv_Reconstruction *libmv_reconstruction)
double libmv_reprojectionError(const libmv_Reconstruction *libmv_reconstruction)
void libmv_reconstructionDestroy(libmv_Reconstruction *libmv_reconstruction)
int libmv_reconstructionIsValid(libmv_Reconstruction *libmv_reconstruction)
int libmv_reprojectionCameraForImage(const libmv_Reconstruction *libmv_reconstruction, int image, double mat[4][4])
libmv_Reconstruction * libmv_solveModal(const libmv_Tracks *libmv_tracks, const libmv_CameraIntrinsicsOptions *libmv_camera_intrinsics_options, const libmv_ReconstructionOptions *libmv_reconstruction_options, reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
int libmv_reprojectionPointForTrack(const libmv_Reconstruction *libmv_reconstruction, int track, double pos[3])
#define SET_DISTORTION_FLAG_CHECKED(type, coefficient)
double libmv_reprojectionErrorForTrack(const libmv_Reconstruction *libmv_reconstruction, int track)
void(* reconstruct_progress_update_cb)(void *customdata, double progress, const char *message)
@ LIBMV_REFINE_FOCAL_LENGTH
@ LIBMV_REFINE_PRINCIPAL_POINT
struct libmv_Tracks libmv_Tracks
Definition: intern/tracks.h:11
const ProjectiveReconstruction & reconstruction
Definition: intersect.cc:198
#define LG
@ BUNDLE_NO_TRANSLATION
Definition: bundle.h:118
@ BUNDLE_NO_CONSTRAINTS
Definition: bundle.h:117
std::vector< ElementType, Eigen::aligned_allocator< ElementType > > vector
void ModalSolver(const Tracks &tracks, EuclideanReconstruction *reconstruction, ProgressUpdateCallback *update_callback)
@ BUNDLE_FOCAL_LENGTH
Definition: bundle.h:102
@ BUNDLE_NO_INTRINSICS
Definition: bundle.h:100
@ BUNDLE_PRINCIPAL_POINT
Definition: bundle.h:103
void SelectKeyframesBasedOnGRICAndVariance(const Tracks &_tracks, const CameraIntrinsics &intrinsics, vector< int > &keyframes)
bool EuclideanReconstructTwoFrames(const vector< Marker > &markers, EuclideanReconstruction *reconstruction)
void EuclideanScaleToUnity(EuclideanReconstruction *reconstruction)
void EuclideanBundle(const Tracks &tracks, EuclideanReconstruction *reconstruction)
Definition: bundle.cc:650
void EuclideanCompleteReconstruction(const Tracks &tracks, EuclideanReconstruction *reconstruction, ProgressUpdateCallback *update_callback)
Eigen::Vector3d Vec3
Definition: numeric.h:106
void EuclideanBundleCommonIntrinsics(const Tracks &tracks, const int bundle_intrinsics, const int bundle_constraints, EuclideanReconstruction *reconstruction, CameraIntrinsics *intrinsics, BundleEvaluation *evaluation)
Definition: bundle.cc:661
double EuclideanReprojectionError(const Tracks &image_tracks, const EuclideanReconstruction &reconstruction, const CameraIntrinsics &intrinsics)
@ TANGENTIAL
Definition: polyfill_2d.c:114
CameraIntrinsics * intrinsics
EuclideanReconstruction reconstruction
ListBase tracks
Definition: tracking.c:60
float max
#define LIBMV_OBJECT_NEW(type,...)
Definition: utildefines.h:38
#define LIBMV_OBJECT_DELETE(what, type)
Definition: utildefines.h:41