Blender  V3.3
intern/libmv/libmv/simple_pipeline/pipeline.cc
Go to the documentation of this file.
1 // Copyright (c) 2011 libmv authors.
2 //
3 // Permission is hereby granted, free of charge, to any person obtaining a copy
4 // of this software and associated documentation files (the "Software"), to
5 // deal in the Software without restriction, including without limitation the
6 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
7 // sell copies of the Software, and to permit persons to whom the Software is
8 // furnished to do so, subject to the following conditions:
9 //
10 // The above copyright notice and this permission notice shall be included in
11 // all copies or substantial portions of the Software.
12 //
13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
18 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
19 // IN THE SOFTWARE.
20 
22 
23 #include <cstdio>
24 
25 #include "libmv/logging/logging.h"
32 
33 #ifdef _MSC_VER
34 # define snprintf _snprintf
35 #endif
36 
37 namespace libmv {
38 namespace {
39 
40 // These are "strategy" classes which make it possible to use the same code for
41 // both projective and euclidean reconstruction.
42 // FIXME(MatthiasF): OOP would achieve the same goal while avoiding
43 // template bloat and making interface changes much easier.
44 struct EuclideanPipelineRoutines {
45  typedef EuclideanReconstruction Reconstruction;
46  typedef EuclideanCamera Camera;
47  typedef EuclideanPoint Point;
48 
49  static void Bundle(const Tracks& tracks,
50  EuclideanReconstruction* reconstruction) {
52  }
53 
54  static bool Resect(const vector<Marker>& markers,
55  EuclideanReconstruction* reconstruction,
56  bool final_pass) {
57  return EuclideanResect(markers, reconstruction, final_pass);
58  }
59 
60  static bool Intersect(const vector<Marker>& markers,
61  EuclideanReconstruction* reconstruction) {
63  }
64 
65  static Marker ProjectMarker(const EuclideanPoint& point,
66  const EuclideanCamera& camera,
67  const CameraIntrinsics& intrinsics) {
68  Vec3 projected = camera.R * point.X + camera.t;
69  projected /= projected(2);
70 
71  Marker reprojected_marker;
72  intrinsics.ApplyIntrinsics(projected(0),
73  projected(1),
74  &reprojected_marker.x,
75  &reprojected_marker.y);
76 
77  reprojected_marker.image = camera.image;
78  reprojected_marker.track = point.track;
79  return reprojected_marker;
80  }
81 };
82 
83 struct ProjectivePipelineRoutines {
84  typedef ProjectiveReconstruction Reconstruction;
85  typedef ProjectiveCamera Camera;
86  typedef ProjectivePoint Point;
87 
88  static void Bundle(const Tracks& tracks,
89  ProjectiveReconstruction* reconstruction) {
91  }
92 
93  static bool Resect(const vector<Marker>& markers,
94  ProjectiveReconstruction* reconstruction,
95  bool final_pass) {
96  (void)final_pass; // Ignored.
97 
99  }
100 
101  static bool Intersect(const vector<Marker>& markers,
102  ProjectiveReconstruction* reconstruction) {
104  }
105 
106  static Marker ProjectMarker(const ProjectivePoint& point,
107  const ProjectiveCamera& camera,
108  const CameraIntrinsics& intrinsics) {
109  Vec3 projected = camera.P * point.X;
110  projected /= projected(2);
111 
112  Marker reprojected_marker;
113  intrinsics.ApplyIntrinsics(projected(0),
114  projected(1),
115  &reprojected_marker.x,
116  &reprojected_marker.y);
117 
118  reprojected_marker.image = camera.image;
119  reprojected_marker.track = point.track;
120  return reprojected_marker;
121  }
122 };
123 
124 } // namespace
125 
127  ProgressUpdateCallback* update_callback,
128  double progress,
129  const char* step = NULL) {
130  if (update_callback) {
131  char message[256];
132 
133  if (step)
134  snprintf(message,
135  sizeof(message),
136  "Completing solution %d%% | %s",
137  (int)(progress * 100),
138  step);
139  else
140  snprintf(message,
141  sizeof(message),
142  "Completing solution %d%%",
143  (int)(progress * 100));
144 
145  update_callback->invoke(progress, message);
146  }
147 }
148 
149 template <typename PipelineRoutines>
151  const Tracks& tracks,
152  typename PipelineRoutines::Reconstruction* reconstruction,
153  ProgressUpdateCallback* update_callback = NULL) {
154  int max_track = tracks.MaxTrack();
155  int max_image = tracks.MaxImage();
156  int num_resects = -1;
157  int num_intersects = -1;
158  int tot_resects = 0;
159  LG << "Max track: " << max_track;
160  LG << "Max image: " << max_image;
161  LG << "Number of markers: " << tracks.NumMarkers();
162  while (num_resects != 0 || num_intersects != 0) {
163  // Do all possible intersections.
164  num_intersects = 0;
165  for (int track = 0; track <= max_track; ++track) {
166  if (reconstruction->PointForTrack(track)) {
167  LG << "Skipping point: " << track;
168  continue;
169  }
170  vector<Marker> all_markers = tracks.MarkersForTrack(track);
171  LG << "Got " << all_markers.size() << " markers for track " << track;
172 
173  vector<Marker> reconstructed_markers;
174  for (int i = 0; i < all_markers.size(); ++i) {
175  if (reconstruction->CameraForImage(all_markers[i].image)) {
176  reconstructed_markers.push_back(all_markers[i]);
177  }
178  }
179  LG << "Got " << reconstructed_markers.size()
180  << " reconstructed markers for track " << track;
181  if (reconstructed_markers.size() >= 2) {
182  CompleteReconstructionLogProgress(update_callback,
183  (double)tot_resects / (max_image));
184  if (PipelineRoutines::Intersect(reconstructed_markers,
185  reconstruction)) {
186  num_intersects++;
187  LG << "Ran Intersect() for track " << track;
188  } else {
189  LG << "Failed Intersect() for track " << track;
190  }
191  }
192  }
193  if (num_intersects) {
195  update_callback, (double)tot_resects / (max_image), "Bundling...");
196  PipelineRoutines::Bundle(tracks, reconstruction);
197  LG << "Ran Bundle() after intersections.";
198  }
199  LG << "Did " << num_intersects << " intersects.";
200 
201  // Do all possible resections.
202  num_resects = 0;
203  for (int image = 0; image <= max_image; ++image) {
205  LG << "Skipping frame: " << image;
206  continue;
207  }
208  vector<Marker> all_markers = tracks.MarkersInImage(image);
209  LG << "Got " << all_markers.size() << " markers for image " << image;
210 
211  vector<Marker> reconstructed_markers;
212  for (int i = 0; i < all_markers.size(); ++i) {
213  if (reconstruction->PointForTrack(all_markers[i].track)) {
214  reconstructed_markers.push_back(all_markers[i]);
215  }
216  }
217  LG << "Got " << reconstructed_markers.size()
218  << " reconstructed markers for image " << image;
219  if (reconstructed_markers.size() >= 5) {
220  CompleteReconstructionLogProgress(update_callback,
221  (double)tot_resects / (max_image));
222  if (PipelineRoutines::Resect(
223  reconstructed_markers, reconstruction, false)) {
224  num_resects++;
225  tot_resects++;
226  LG << "Ran Resect() for image " << image;
227  } else {
228  LG << "Failed Resect() for image " << image;
229  }
230  }
231  }
232  if (num_resects) {
234  update_callback, (double)tot_resects / (max_image), "Bundling...");
235  PipelineRoutines::Bundle(tracks, reconstruction);
236  }
237  LG << "Did " << num_resects << " resects.";
238  }
239 
240  // One last pass...
241  num_resects = 0;
242  for (int image = 0; image <= max_image; ++image) {
244  LG << "Skipping frame: " << image;
245  continue;
246  }
247  vector<Marker> all_markers = tracks.MarkersInImage(image);
248 
249  vector<Marker> reconstructed_markers;
250  for (int i = 0; i < all_markers.size(); ++i) {
251  if (reconstruction->PointForTrack(all_markers[i].track)) {
252  reconstructed_markers.push_back(all_markers[i]);
253  }
254  }
255  if (reconstructed_markers.size() >= 5) {
256  CompleteReconstructionLogProgress(update_callback,
257  (double)tot_resects / (max_image));
258  if (PipelineRoutines::Resect(
259  reconstructed_markers, reconstruction, true)) {
260  num_resects++;
261  LG << "Ran final Resect() for image " << image;
262  } else {
263  LG << "Failed final Resect() for image " << image;
264  }
265  }
266  }
267  if (num_resects) {
269  update_callback, (double)tot_resects / (max_image), "Bundling...");
270  PipelineRoutines::Bundle(tracks, reconstruction);
271  }
272 }
273 
274 template <typename PipelineRoutines>
276  const Tracks& image_tracks,
277  const typename PipelineRoutines::Reconstruction& reconstruction,
278  const CameraIntrinsics& intrinsics) {
279  int num_skipped = 0;
280  int num_reprojected = 0;
281  double total_error = 0.0;
282  vector<Marker> markers = image_tracks.AllMarkers();
283  for (int i = 0; i < markers.size(); ++i) {
284  double weight = markers[i].weight;
285  const typename PipelineRoutines::Camera* camera =
287  const typename PipelineRoutines::Point* point =
289  if (!camera || !point || weight == 0.0) {
290  num_skipped++;
291  continue;
292  }
293  num_reprojected++;
294 
295  Marker reprojected_marker =
296  PipelineRoutines::ProjectMarker(*point, *camera, intrinsics);
297  double ex = (reprojected_marker.x - markers[i].x) * weight;
298  double ey = (reprojected_marker.y - markers[i].y) * weight;
299 
300  const int N = 100;
301  char line[N];
302  snprintf(line,
303  N,
304  "image %-3d track %-3d "
305  "x %7.1f y %7.1f "
306  "rx %7.1f ry %7.1f "
307  "ex %7.1f ey %7.1f"
308  " e %7.1f",
309  markers[i].image,
310  markers[i].track,
311  markers[i].x,
312  markers[i].y,
313  reprojected_marker.x,
314  reprojected_marker.y,
315  ex,
316  ey,
317  sqrt(ex * ex + ey * ey));
318  VLOG(1) << line;
319 
320  total_error += sqrt(ex * ex + ey * ey);
321  }
322  LG << "Skipped " << num_skipped << " markers.";
323  LG << "Reprojected " << num_reprojected << " markers.";
324  LG << "Total error: " << total_error << " px";
325  LG << "Average error: " << (total_error / num_reprojected) << " px";
326  return total_error / num_reprojected;
327 }
328 
329 double EuclideanReprojectionError(const Tracks& image_tracks,
331  const CameraIntrinsics& intrinsics) {
332  return InternalReprojectionError<EuclideanPipelineRoutines>(
333  image_tracks, reconstruction, intrinsics);
334 }
335 
337  const Tracks& image_tracks,
339  const CameraIntrinsics& intrinsics) {
340  return InternalReprojectionError<ProjectivePipelineRoutines>(
341  image_tracks, reconstruction, intrinsics);
342 }
343 
346  ProgressUpdateCallback* update_callback) {
347  InternalCompleteReconstruction<EuclideanPipelineRoutines>(
348  tracks, reconstruction, update_callback);
349 }
350 
353  InternalCompleteReconstruction<ProjectivePipelineRoutines>(tracks,
355 }
356 
357 void InvertIntrinsicsForTracks(const Tracks& raw_tracks,
358  const CameraIntrinsics& camera_intrinsics,
359  Tracks* calibrated_tracks) {
360  vector<Marker> markers = raw_tracks.AllMarkers();
361  for (int i = 0; i < markers.size(); ++i) {
362  camera_intrinsics.InvertIntrinsics(
363  markers[i].x, markers[i].y, &(markers[i].x), &(markers[i].y));
364  }
365  *calibrated_tracks = Tracks(markers);
366 }
367 
368 } // namespace libmv
sqrt(x)+1/max(0
#define snprintf
Definition: BLI_winstuff.h:53
struct Camera Camera
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei GLfloat GLfloat GLfloat GLfloat const GLubyte *bitmap _GL_VOID_RET _GL_VOID GLenum const void *lists _GL_VOID_RET _GL_VOID const GLdouble *equation _GL_VOID_RET _GL_VOID GLdouble GLdouble blue _GL_VOID_RET _GL_VOID GLfloat GLfloat blue _GL_VOID_RET _GL_VOID GLint GLint blue _GL_VOID_RET _GL_VOID GLshort GLshort blue _GL_VOID_RET _GL_VOID GLubyte GLubyte blue _GL_VOID_RET _GL_VOID GLuint GLuint blue _GL_VOID_RET _GL_VOID GLushort GLushort blue _GL_VOID_RET _GL_VOID GLbyte GLbyte GLbyte alpha _GL_VOID_RET _GL_VOID GLdouble GLdouble GLdouble alpha _GL_VOID_RET _GL_VOID GLfloat GLfloat GLfloat alpha _GL_VOID_RET _GL_VOID GLint GLint GLint alpha _GL_VOID_RET _GL_VOID GLshort GLshort GLshort alpha _GL_VOID_RET _GL_VOID GLubyte GLubyte GLubyte alpha _GL_VOID_RET _GL_VOID GLuint GLuint GLuint alpha _GL_VOID_RET _GL_VOID GLushort GLushort GLushort alpha _GL_VOID_RET _GL_VOID GLenum mode _GL_VOID_RET _GL_VOID GLint y
in reality light always falls off quadratically Particle Retrieve the data of the particle that spawned the object for example to give variation to multiple instances of an object Point Retrieve information about points in a point cloud Retrieve the edges of an object as it appears to Cycles topology will always appear triangulated Convert a blackbody temperature to an RGB value Normal Generate a perturbed normal from an RGB normal map image Typically used for faking highly detailed surfaces Generate an OSL shader from a file or text data block Image Sample an image file as a texture Sky Generate a procedural sky texture Noise Generate fractal Perlin noise Wave Generate procedural bands or rings with noise Voronoi Generate Worley noise based on the distance to random points Typically used to generate textures such as or biological cells Brick Generate a procedural texture producing bricks Texture Retrieve multiple types of texture coordinates nTypically used as inputs for texture nodes Vector Convert a point
in reality light always falls off quadratically Particle Retrieve the data of the particle that spawned the object for example to give variation to multiple instances of an object Point Retrieve information about points in a point cloud Retrieve the edges of an object as it appears to Cycles topology will always appear triangulated Convert a blackbody temperature to an RGB value Normal Generate a perturbed normal from an RGB normal map image Typically used for faking highly detailed surfaces Generate an OSL shader from a file or text data block Image Sample an image file as a texture Sky Generate a procedural sky texture Noise Generate fractal Perlin noise Wave Generate procedural bands or rings with noise Voronoi Generate Worley noise based on the distance to random points Typically used to generate textures such as or biological cells Brick Generate a procedural texture producing bricks Texture Retrieve multiple types of texture coordinates nTypically used as inputs for texture nodes Vector Convert a or normal between camera
DBVT_INLINE bool Intersect(const btDbvtAabbMm &a, const btDbvtAabbMm &b)
Definition: btDbvt.h:621
virtual void InvertIntrinsics(double image_x, double image_y, double *normalized_x, double *normalized_y) const =0
virtual void invoke(double progress, const char *message)=0
ProjectivePoint * PointForTrack(int track)
Returns a pointer to the point corresponding to track.
ProjectiveCamera * CameraForImage(int image)
Returns a pointer to the camera corresponding to image.
vector< Marker > AllMarkers() const
Returns all the markers.
SyclQueue void void size_t num_bytes void
depth_tx normal_tx diffuse_light_tx specular_light_tx volume_light_tx environment_tx ambient_occlusion_tx aov_value_tx in_weight_img image(1, GPU_R32F, Qualifier::WRITE, ImageType::FLOAT_2D_ARRAY, "out_weight_img") .image(3
const vector< Marker > & markers
const ProjectiveReconstruction & reconstruction
Definition: intersect.cc:198
#define LG
#define VLOG(severity)
Definition: log.h:37
#define N
double ProjectiveReprojectionError(const Tracks &image_tracks, const ProjectiveReconstruction &reconstruction, const CameraIntrinsics &intrinsics)
void ProjectiveBundle(const Tracks &, ProjectiveReconstruction *)
Definition: bundle.cc:852
bool ProjectiveResect(const vector< Marker > &markers, ProjectiveReconstruction *reconstruction)
Definition: resect.cc:224
bool EuclideanResect(const vector< Marker > &markers, EuclideanReconstruction *reconstruction, bool final_pass)
Definition: resect.cc:94
static void CompleteReconstructionLogProgress(ProgressUpdateCallback *update_callback, double progress, const char *step=NULL)
void EuclideanBundle(const Tracks &tracks, EuclideanReconstruction *reconstruction)
Definition: bundle.cc:650
void EuclideanCompleteReconstruction(const Tracks &tracks, EuclideanReconstruction *reconstruction, ProgressUpdateCallback *update_callback)
bool EuclideanIntersect(const vector< Marker > &markers, EuclideanReconstruction *reconstruction)
Definition: intersect.cc:69
Eigen::Vector3d Vec3
Definition: numeric.h:106
double InternalReprojectionError(const Tracks &image_tracks, const typename PipelineRoutines::Reconstruction &reconstruction, const CameraIntrinsics &intrinsics)
double EuclideanReprojectionError(const Tracks &image_tracks, const EuclideanReconstruction &reconstruction, const CameraIntrinsics &intrinsics)
void InvertIntrinsicsForTracks(const Tracks &raw_tracks, const CameraIntrinsics &camera_intrinsics, Tracks *calibrated_tracks)
bool ProjectiveIntersect(const vector< Marker > &markers, ProjectiveReconstruction *reconstruction)
Definition: intersect.cc:203
void InternalCompleteReconstruction(const Tracks &tracks, typename PipelineRoutines::Reconstruction *reconstruction, ProgressUpdateCallback *update_callback=NULL)
void ProjectiveCompleteReconstruction(const Tracks &tracks, ProjectiveReconstruction *reconstruction)
ListBase tracks
Definition: tracking.c:60