Blender  V3.3
libmv/autotrack/autotrack.h
Go to the documentation of this file.
1 // Copyright (c) 2014 libmv authors.
2 //
3 // Permission is hereby granted, free of charge, to any person obtaining a copy
4 // of this software and associated documentation files (the "Software"), to
5 // deal in the Software without restriction, including without limitation the
6 // rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
7 // sell copies of the Software, and to permit persons to whom the Software is
8 // furnished to do so, subject to the following conditions:
9 //
10 // The above copyright notice and this permission notice shall be included in
11 // all copies or substantial portions of the Software.
12 //
13 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
18 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
19 // IN THE SOFTWARE.
20 //
21 // Author: mierle@gmail.com (Keir Mierle)
22 
23 #ifndef LIBMV_AUTOTRACK_AUTOTRACK_H_
24 #define LIBMV_AUTOTRACK_AUTOTRACK_H_
25 
26 #include "libmv/autotrack/region.h"
27 #include "libmv/autotrack/tracks.h"
29 
30 namespace libmv {
31 class CameraIntrinsics;
32 };
33 
34 namespace mv {
35 
39 
40 struct FrameAccessor;
41 class OperationListener;
42 
43 // The coordinator of all tracking operations; keeps track of all state
44 // relating to tracking and reconstruction; for example, 2D tracks and motion
45 // models, reconstructed cameras, points, and planes; tracking settings; etc.
46 //
47 // Typical usage for full autotrack:
48 //
49 // AutoTrack auto_track(image_accessor);
50 // auto_track.SetNumFramesInClip(0, 10);
51 // auto_track.SetNumFramesInClip(1, 54);
52 // auto_track.AutoTrack()
53 //
54 // It is also possible to specify options to control the reconstruction.
55 // Furthermore, the individual methods of reconstruction are exposed to make it
56 // possible to interact with the pipeline as it runs. For example, to track one
57 // marker across frames,
58 //
59 // AutoTrack auto_track(image_accessor);
60 // auto_track.SetNumFramesInClip(0, 10);
61 // auto_track.SetNumFramesInClip(1, 54);
62 // auto_track.AddMarker(...);
63 // auto_track.TrackMarkerToFrame(int clip1, int frame1,
64 // int clip2, int frame2,
65 // options?)
66 //
67 class AutoTrack {
68  public:
69  struct Options {
70  // Default configuration for 2D tracking when calling TrackMarkerToFrame().
72 
73  // Default search window for region tracking, in absolute frame pixels.
75  };
76 
77  AutoTrack(FrameAccessor* frame_accessor) : frame_accessor_(frame_accessor) {}
78 
79  // Marker manipulation.
80  // Clip manipulation.
81 
82  // Set the number of clips. These clips will get accessed from the frame
83  // accessor, matches between frames found, and a reconstruction created.
84  // void SetNumFrames(int clip, int num_frames);
85 
86  // Tracking & Matching
87 
88  // Find the marker for the track in the frame indicated by the marker.
89  // Caller maintains ownership of *result and *tracked_marker.
90  bool TrackMarker(Marker* tracked_marker,
92  const TrackRegionOptions* track_options = NULL);
93 
94  // Wrapper around Tracks API; however these may add additional processing.
95  void AddMarker(const Marker& tracked_marker);
96  void SetMarkers(vector<Marker>* markers);
97  bool GetMarker(int clip, int frame, int track, Marker* marker) const;
98 
99  // TODO(keir): Implement frame matching! This could be very cool for loop
100  // closing and connecting across clips.
101  // void MatchFrames(int clip1, int frame1, int clip2, int frame2) {}
102 
103  // Wrapper around the Reconstruction API.
104  // Returns the new ID.
106  (void)intrinsics;
107  return 0;
108  } // XXX
109  int SetClipIntrinsics(int clip, int intrinsics) {
110  (void)clip;
111  (void)intrinsics;
112  return 0;
113  } // XXX
114 
115  enum Motion {
118  };
119  int SetClipMotion(int clip, Motion motion) {
120  (void)clip;
121  (void)motion;
122  return 0;
123  } // XXX
124 
125  // Decide what to refine for the given intrinsics. bundle_options is from
126  // bundle.h (e.g. BUNDLE_FOCAL_LENGTH | BUNDLE_RADIAL_K1).
127  void SetIntrinsicsRefine(int intrinsics, int bundle_options) {
128  (void)intrinsics;
129  (void)bundle_options;
130  } // XXX
131 
132  // Keyframe read/write.
133  struct ClipFrame {
134  int clip;
135  int frame;
136  };
137  const vector<ClipFrame>& keyframes() { return keyframes_; }
138  void ClearKeyframes() { keyframes_.clear(); }
139  void SetKeyframes(const vector<ClipFrame>& keyframes) {
140  keyframes_ = keyframes;
141  }
142 
143  // What about reporting what happened? -- callbacks; maybe result struct.
144  void Reconstruct();
145 
146  // Detect and track in 2D.
149  };
151 
154  int clip, int frame, const DetectFeaturesInFrameOptions* options = NULL) {
155  (void)clip;
156  (void)frame;
157  (void)options;
158  } // XXX
159 
160  // Does not take ownership of the given listener, but keeps a reference to it.
161  void AddListener(OperationListener* listener) { (void)listener; } // XXX
162 
163  // Create the initial reconstruction,
164  // void FindInitialReconstruction();
165 
166  // State machine
167  //
168  // Question: Have explicit state? Or determine state from existing data?
169  // Conclusion: Determine state from existing data.
170  //
171  // Preliminary state thoughts
172  //
173  // No tracks or markers
174  // - Tracks empty.
175  //
176  // Initial tracks found
177  // - All images have at least 5 tracks
178  //
179  // Ran RANSAC on tracks to mark inliers / outliers.
180  // - All images have at least 8 "inlier" tracks
181  //
182  // Detector matching run to close loops and match across clips
183  // - At least 5 matching tracks between clips
184  //
185  // Initial reconstruction found (2 frames)?
186  // - There exists two cameras with intrinsics / extrinsics
187  //
188  // Preliminary reconstruction finished
189  // - Poses for all frames in all clips estimated.
190  //
191  // Final reconstruction finished
192  // - Final reconstruction bundle adjusted.
193 
194  // For now, expose options directly. In the future this may change.
196 
197  private:
198  bool Log();
199  bool Progress();
200  bool Cancelled() { return false; }
201 
202  Tracks tracks_; // May be normalized camera coordinates or raw pixels.
203  // Reconstruction reconstruction_;
204 
205  // TODO(keir): Add the motion models here.
206  // vector<MotionModel> motion_models_;
207 
208  // TODO(keir): Should num_clips and num_frames get moved to FrameAccessor?
209  // TODO(keir): What about masking for clips and frames to prevent various
210  // things like reconstruction or tracking from happening on certain frames?
211  FrameAccessor* frame_accessor_;
212  // int num_clips_;
213  // vector<int> num_frames_; // Indexed by clip.
214 
215  // The intrinsics for each clip, assuming each clip has fixed intrinsics.
216  // TODO(keir): Decide what the semantics should be for varying focal length.
217  vector<int> clip_intrinsics_;
218 
219  vector<ClipFrame> keyframes_;
220 };
221 
222 } // namespace mv
223 
224 #endif // LIBMV_AUTOTRACK_AUTOTRACK_H_
void DetectAndTrack(const DetectAndTrackOptions &options)
void SetMarkers(vector< Marker > *markers)
void SetKeyframes(const vector< ClipFrame > &keyframes)
void Reconstruct()
int SetClipMotion(int clip, Motion motion)
bool GetMarker(int clip, int frame, int track, Marker *marker) const
int SetClipIntrinsics(int clip, int intrinsics)
void AddMarker(const Marker &tracked_marker)
void AddListener(OperationListener *listener)
int AddCameraIntrinsics(CameraIntrinsics *intrinsics)
const vector< ClipFrame > & keyframes()
void SetIntrinsicsRefine(int intrinsics, int bundle_options)
bool TrackMarker(Marker *tracked_marker, TrackRegionResult *result, const TrackRegionOptions *track_options=NULL)
AutoTrack(FrameAccessor *frame_accessor)
void DetectFeaturesInFrame(int clip, int frame, const DetectFeaturesInFrameOptions *options=NULL)
SyclQueue void void size_t num_bytes void
const vector< Marker > & markers