OpenPose  1.7.0
The first real-time multi-person system to jointly detect human body, hand, facial, and foot keypoints
wrapperAuxiliary.hpp
Go to the documentation of this file.
1 #ifndef OPENPOSE_WRAPPER_WRAPPER_AUXILIARY_HPP
2 #define OPENPOSE_WRAPPER_WRAPPER_AUXILIARY_HPP
3 
4 #include <openpose/thread/headers.hpp>
5 #include <openpose/wrapper/enumClasses.hpp>
6 #include <openpose/wrapper/wrapperStructExtra.hpp>
7 #include <openpose/wrapper/wrapperStructFace.hpp>
8 #include <openpose/wrapper/wrapperStructGui.hpp>
9 #include <openpose/wrapper/wrapperStructHand.hpp>
10 #include <openpose/wrapper/wrapperStructInput.hpp>
11 #include <openpose/wrapper/wrapperStructOutput.hpp>
12 #include <openpose/wrapper/wrapperStructPose.hpp>
13 
14 namespace op
15 {
29  OP_API void wrapperConfigureSanityChecks(
30  WrapperStructPose& wrapperStructPose, const WrapperStructFace& wrapperStructFace,
31  const WrapperStructHand& wrapperStructHand, const WrapperStructExtra& wrapperStructExtra,
32  const WrapperStructInput& wrapperStructInput, const WrapperStructOutput& wrapperStructOutput,
33  const WrapperStructGui& wrapperStructGui, const bool renderOutput, const bool userInputAndPreprocessingWsEmpty,
34  const bool userOutputWsEmpty, const std::shared_ptr<Producer>& producerSharedPtr,
35  const ThreadManagerMode threadManagerMode);
36 
45  OP_API void threadIdPP(unsigned long long& threadId, const bool multiThreadEnabled);
46 
53  template<typename TDatum,
54  typename TDatums = std::vector<std::shared_ptr<TDatum>>,
55  typename TDatumsSP = std::shared_ptr<TDatums>,
56  typename TWorker = std::shared_ptr<Worker<TDatumsSP>>>
57  void configureThreadManager(
58  ThreadManager<TDatumsSP>& threadManager, const bool multiThreadEnabled,
59  const ThreadManagerMode threadManagerMode, const WrapperStructPose& wrapperStructPose,
60  const WrapperStructFace& wrapperStructFace, const WrapperStructHand& wrapperStructHand,
61  const WrapperStructExtra& wrapperStructExtra, const WrapperStructInput& wrapperStructInput,
62  const WrapperStructOutput& wrapperStructOutput, const WrapperStructGui& wrapperStructGui,
63  const std::array<std::vector<TWorker>, int(WorkerType::Size)>& userWs,
64  const std::array<bool, int(WorkerType::Size)>& userWsOnNewThread);
65 
70  template<typename TDatum,
71  typename TDatums = std::vector<std::shared_ptr<TDatum>>,
72  typename TDatumsSP = std::shared_ptr<TDatums>>
73  void createMultiviewTDatum(
74  TDatumsSP& tDatumsSP, unsigned long long& frameCounter,
75  const CameraParameterReader& cameraParameterReader, const void* const cvMatPtr);
76 }
77 
78 
79 
80 
81 
82 // Implementation
83 #include <openpose/3d/headers.hpp>
84 #include <openpose/core/headers.hpp>
85 #include <openpose/face/headers.hpp>
86 #include <openpose/filestream/headers.hpp>
87 #include <openpose/gpu/gpu.hpp>
88 #include <openpose/gui/headers.hpp>
89 #include <openpose/hand/headers.hpp>
90 #include <openpose/pose/headers.hpp>
91 #include <openpose/producer/headers.hpp>
92 #include <openpose/tracking/headers.hpp>
93 #include <openpose/utilities/fileSystem.hpp>
94 #include <openpose/utilities/standard.hpp>
95 namespace op
96 {
97  template<typename TDatum, typename TDatums, typename TDatumsSP, typename TWorker>
98  void configureThreadManager(
99  ThreadManager<TDatumsSP>& threadManager, const bool multiThreadEnabledTemp,
100  const ThreadManagerMode threadManagerMode, const WrapperStructPose& wrapperStructPoseTemp,
101  const WrapperStructFace& wrapperStructFace, const WrapperStructHand& wrapperStructHand,
102  const WrapperStructExtra& wrapperStructExtra, const WrapperStructInput& wrapperStructInput,
103  const WrapperStructOutput& wrapperStructOutput, const WrapperStructGui& wrapperStructGui,
104  const std::array<std::vector<TWorker>, int(WorkerType::Size)>& userWs,
105  const std::array<bool, int(WorkerType::Size)>& userWsOnNewThread)
106  {
107  try
108  {
109  opLog("Running configureThreadManager...", Priority::Normal);
110 
111  // Create producer
112  auto producerSharedPtr = createProducer(
113  wrapperStructInput.producerType, wrapperStructInput.producerString.getStdString(),
114  wrapperStructInput.cameraResolution, wrapperStructInput.cameraParameterPath.getStdString(),
115  wrapperStructInput.undistortImage, wrapperStructInput.numberViews);
116 
117  // Editable arguments
118  auto wrapperStructPose = wrapperStructPoseTemp;
119  auto multiThreadEnabled = multiThreadEnabledTemp;
120 
121  // User custom workers
122  const auto& userInputWs = userWs[int(WorkerType::Input)];
123  const auto& userPreProcessingWs = userWs[int(WorkerType::PreProcessing)];
124  const auto& userPostProcessingWs = userWs[int(WorkerType::PostProcessing)];
125  const auto& userOutputWs = userWs[int(WorkerType::Output)];
126  const auto userInputWsOnNewThread = userWsOnNewThread[int(WorkerType::Input)];
127  const auto userPreProcessingWsOnNewThread = userWsOnNewThread[int(WorkerType::PreProcessing)];
128  const auto userPostProcessingWsOnNewThread = userWsOnNewThread[int(WorkerType::PostProcessing)];
129  const auto userOutputWsOnNewThread = userWsOnNewThread[int(WorkerType::Output)];
130 
131  // Video seek
132  const auto spVideoSeek = std::make_shared<std::pair<std::atomic<bool>, std::atomic<int>>>();
133  // It cannot be directly included in the constructor (compiler error for copying std::atomic)
134  spVideoSeek->first = false;
135  spVideoSeek->second = 0;
136 
137  // Required parameters
138  const auto gpuMode = getGpuMode();
139  const auto renderModePose = (
140  wrapperStructPose.renderMode != RenderMode::Auto
141  ? wrapperStructPose.renderMode
142  : (gpuMode == GpuMode::Cuda ? RenderMode::Gpu : RenderMode::Cpu));
143  const auto renderModeFace = (
144  wrapperStructFace.renderMode != RenderMode::Auto
145  ? wrapperStructFace.renderMode
146  : (gpuMode == GpuMode::Cuda ? RenderMode::Gpu : RenderMode::Cpu));
147  const auto renderModeHand = (
148  wrapperStructHand.renderMode != RenderMode::Auto
149  ? wrapperStructHand.renderMode
150  : (gpuMode == GpuMode::Cuda ? RenderMode::Gpu : RenderMode::Cpu));
151  const auto renderOutput = renderModePose != RenderMode::None
152  || renderModeFace != RenderMode::None
153  || renderModeHand != RenderMode::None;
154  const bool renderOutputGpu = renderModePose == RenderMode::Gpu
155  || (wrapperStructFace.enable && renderModeFace == RenderMode::Gpu)
156  || (wrapperStructHand.enable && renderModeHand == RenderMode::Gpu);
157  const bool renderFace = wrapperStructFace.enable && renderModeFace != RenderMode::None;
158  const bool renderHand = wrapperStructHand.enable && renderModeHand != RenderMode::None;
159  const bool renderHandGpu = wrapperStructHand.enable && renderModeHand == RenderMode::Gpu;
160  opLog("renderModePose = " + std::to_string(int(renderModePose)), Priority::Normal);
161  opLog("renderModeFace = " + std::to_string(int(renderModeFace)), Priority::Normal);
162  opLog("renderModeHand = " + std::to_string(int(renderModeHand)), Priority::Normal);
163  opLog("renderOutput = " + std::to_string(int(renderOutput)), Priority::Normal);
164  opLog("renderOutputGpu = " + std::to_string(int(renderOutput)), Priority::Normal);
165  opLog("renderFace = " + std::to_string(int(renderFace)), Priority::Normal);
166  opLog("renderHand = " + std::to_string(int(renderHand)), Priority::Normal);
167  opLog("renderHandGpu = " + std::to_string(int(renderHandGpu)), Priority::Normal);
168 
169  // Check no wrong/contradictory flags enabled
170  const bool userInputAndPreprocessingWsEmpty = userInputWs.empty() && userPreProcessingWs.empty();
171  const bool userOutputWsEmpty = userOutputWs.empty();
172  wrapperConfigureSanityChecks(
173  wrapperStructPose, wrapperStructFace, wrapperStructHand, wrapperStructExtra, wrapperStructInput,
174  wrapperStructOutput, wrapperStructGui, renderOutput, userInputAndPreprocessingWsEmpty,
175  userOutputWsEmpty, producerSharedPtr, threadManagerMode);
176  opLog("userInputAndPreprocessingWsEmpty = " + std::to_string(int(userInputAndPreprocessingWsEmpty)),
177  Priority::Normal);
178  opLog("userOutputWsEmpty = " + std::to_string(int(userOutputWsEmpty)), Priority::Normal);
179 
180  // Get number threads
181  auto numberGpuThreads = wrapperStructPose.gpuNumber;
182  auto gpuNumberStart = wrapperStructPose.gpuNumberStart;
183  opLog("numberGpuThreads = " + std::to_string(numberGpuThreads), Priority::Normal);
184  opLog("gpuNumberStart = " + std::to_string(gpuNumberStart), Priority::Normal);
185  // CPU --> 1 thread or no pose extraction
186  if (gpuMode == GpuMode::NoGpu)
187  {
188  numberGpuThreads = (wrapperStructPose.gpuNumber == 0 ? 0 : 1);
189  gpuNumberStart = 0;
190  // Disabling multi-thread makes the code 400 ms faster (2.3 sec vs. 2.7 in i7-6850K)
191  // and fixes the bug that the screen was not properly displayed and only refreshed sometimes
192  // Note: The screen bug could be also fixed by using waitKey(30) rather than waitKey(1)
193  multiThreadEnabled = false;
194  }
195  // GPU --> user picks (<= #GPUs)
196  else
197  {
198  // Get total number GPUs
199  const auto totalGpuNumber = getGpuNumber();
200  // If number GPU < 0 --> set it to all the available GPUs
201  if (numberGpuThreads < 0)
202  {
203  if (totalGpuNumber <= gpuNumberStart)
204  error("Number of initial GPU (`--number_gpu_start`) must be lower than the total number of"
205  " used GPUs (`--number_gpu`)", __LINE__, __FUNCTION__, __FILE__);
206  numberGpuThreads = totalGpuNumber - gpuNumberStart;
207  // Reset initial GPU to 0 (we want them all)
208  // Logging message
209  opLog("Auto-detecting all available GPUs... Detected " + std::to_string(totalGpuNumber)
210  + " GPU(s), using " + std::to_string(numberGpuThreads) + " of them starting at GPU "
211  + std::to_string(gpuNumberStart) + ".", Priority::High);
212  }
213  // Sanity check
214  if (gpuNumberStart + numberGpuThreads > totalGpuNumber)
215  error("Initial GPU selected (`--number_gpu_start`) + number GPUs to use (`--number_gpu`) must"
216  " be lower or equal than the total number of GPUs in your machine ("
217  + std::to_string(gpuNumberStart) + " + "
218  + std::to_string(numberGpuThreads) + " vs. "
219  + std::to_string(totalGpuNumber) + ").",
220  __LINE__, __FUNCTION__, __FILE__);
221  }
222 
223  // Proper format
224  const auto writeImagesCleaned = formatAsDirectory(wrapperStructOutput.writeImages.getStdString());
225  const auto writeKeypointCleaned = formatAsDirectory(wrapperStructOutput.writeKeypoint.getStdString());
226  const auto writeJsonCleaned = formatAsDirectory(wrapperStructOutput.writeJson.getStdString());
227  const auto writeHeatMapsCleaned = formatAsDirectory(wrapperStructOutput.writeHeatMaps.getStdString());
228  const auto modelFolder = formatAsDirectory(wrapperStructPose.modelFolder.getStdString());
229  opLog("writeImagesCleaned = " + writeImagesCleaned, Priority::Normal);
230  opLog("writeKeypointCleaned = " + writeKeypointCleaned, Priority::Normal);
231  opLog("writeJsonCleaned = " + writeJsonCleaned, Priority::Normal);
232  opLog("writeHeatMapsCleaned = " + writeHeatMapsCleaned, Priority::Normal);
233  opLog("modelFolder = " + modelFolder, Priority::Normal);
234 
235  // Common parameters
236  auto finalOutputSize = wrapperStructPose.outputSize;
237  Point<int> producerSize{-1,-1};
238  const auto oPProducer = (producerSharedPtr != nullptr);
239  if (oPProducer)
240  {
241  // 1. Set producer properties
242  const auto displayProducerFpsMode = (wrapperStructInput.realTimeProcessing
243  ? ProducerFpsMode::OriginalFps : ProducerFpsMode::RetrievalFps);
244  producerSharedPtr->setProducerFpsMode(displayProducerFpsMode);
245  producerSharedPtr->set(ProducerProperty::Flip, wrapperStructInput.frameFlip);
246  producerSharedPtr->set(ProducerProperty::Rotation, wrapperStructInput.frameRotate);
247  producerSharedPtr->set(ProducerProperty::AutoRepeat, wrapperStructInput.framesRepeat);
248  // 2. Set finalOutputSize
249  producerSize = Point<int>{(int)producerSharedPtr->get(getCvCapPropFrameWidth()),
250  (int)producerSharedPtr->get(getCvCapPropFrameHeight())};
251  // Set finalOutputSize to input size if desired
252  if (finalOutputSize.x == -1 || finalOutputSize.y == -1)
253  finalOutputSize = producerSize;
254  }
255  opLog("finalOutputSize = [" + std::to_string(finalOutputSize.x) + "," + std::to_string(finalOutputSize.y)
256  + "]", Priority::Normal);
257 
258  // Producer
259  TWorker datumProducerW;
260  if (oPProducer)
261  {
262  const auto datumProducer = std::make_shared<DatumProducer<TDatum>>(
263  producerSharedPtr, wrapperStructInput.frameFirst, wrapperStructInput.frameStep,
264  wrapperStructInput.frameLast, spVideoSeek
265  );
266  datumProducerW = std::make_shared<WDatumProducer<TDatum>>(datumProducer);
267  }
268  else
269  datumProducerW = nullptr;
270 
271  std::vector<std::shared_ptr<PoseExtractorNet>> poseExtractorNets;
272  std::vector<std::shared_ptr<FaceExtractorNet>> faceExtractorNets;
273  std::vector<std::shared_ptr<HandExtractorNet>> handExtractorNets;
274  std::vector<std::shared_ptr<PoseGpuRenderer>> poseGpuRenderers;
275  // CUDA vs. CPU resize
276  std::vector<std::shared_ptr<CvMatToOpOutput>> cvMatToOpOutputs;
277  std::vector<std::shared_ptr<OpOutputToCvMat>> opOutputToCvMats;
278  std::shared_ptr<PoseCpuRenderer> poseCpuRenderer;
279  // Workers
280  TWorker scaleAndSizeExtractorW;
281  TWorker cvMatToOpInputW;
282  TWorker cvMatToOpOutputW;
283  bool addCvMatToOpOutput = renderOutput;
284  bool addCvMatToOpOutputInCpu = addCvMatToOpOutput;
285  std::vector<std::vector<TWorker>> poseExtractorsWs;
286  std::vector<std::vector<TWorker>> poseTriangulationsWs;
287  std::vector<std::vector<TWorker>> jointAngleEstimationsWs;
288  std::vector<TWorker> postProcessingWs;
289  if (numberGpuThreads > 0)
290  {
291  // Get input scales and sizes
292  const auto scaleAndSizeExtractor = std::make_shared<ScaleAndSizeExtractor>(
293  wrapperStructPose.netInputSize, (float)wrapperStructPose.netInputSizeDynamicBehavior, finalOutputSize,
294  wrapperStructPose.scalesNumber, wrapperStructPose.scaleGap);
295  scaleAndSizeExtractorW = std::make_shared<WScaleAndSizeExtractor<TDatumsSP>>(scaleAndSizeExtractor);
296 
297  // Input cvMat to OpenPose input & output format
298  // Note: resize on GPU reduces accuracy about 0.1%
299  bool resizeOnCpu = true;
300  // const auto resizeOnCpu = (wrapperStructPose.poseMode != PoseMode::Enabled);
301  if (resizeOnCpu)
302  {
303  const auto gpuResize = false;
304  const auto cvMatToOpInput = std::make_shared<CvMatToOpInput>(
305  wrapperStructPose.poseModel, gpuResize);
306  cvMatToOpInputW = std::make_shared<WCvMatToOpInput<TDatumsSP>>(cvMatToOpInput);
307  }
308  // Note: We realized that somehow doing it on GPU for any number of GPUs does speedup the whole OP
309  resizeOnCpu = false;
310  addCvMatToOpOutputInCpu = addCvMatToOpOutput
311  && (resizeOnCpu || !renderOutputGpu || wrapperStructPose.poseMode != PoseMode::Enabled
312  // Resize in GPU causing bug
313  || wrapperStructPose.outputSize.x != -1 || wrapperStructPose.outputSize.y != -1);
314  if (addCvMatToOpOutputInCpu)
315  {
316  const auto gpuResize = false;
317  const auto cvMatToOpOutput = std::make_shared<CvMatToOpOutput>(gpuResize);
318  cvMatToOpOutputW = std::make_shared<WCvMatToOpOutput<TDatumsSP>>(cvMatToOpOutput);
319  }
320 
321  // Pose estimators & renderers
322  std::vector<TWorker> cpuRenderers;
323  poseExtractorsWs.clear();
324  poseExtractorsWs.resize(numberGpuThreads);
325  if (wrapperStructPose.poseMode != PoseMode::Disabled)
326  {
327  // Pose estimators
328  for (auto gpuId = 0; gpuId < numberGpuThreads; gpuId++)
329  poseExtractorNets.emplace_back(std::make_shared<PoseExtractorCaffe>(
330  wrapperStructPose.poseModel, modelFolder, gpuId + gpuNumberStart,
331  wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScaleMode,
332  wrapperStructPose.addPartCandidates, wrapperStructPose.maximizePositives,
333  wrapperStructPose.protoTxtPath.getStdString(),
334  wrapperStructPose.caffeModelPath.getStdString(),
335  wrapperStructPose.upsamplingRatio, wrapperStructPose.poseMode == PoseMode::Enabled,
336  wrapperStructPose.enableGoogleLogging
337  ));
338 
339  // Pose renderers
340  if (renderOutputGpu || renderModePose == RenderMode::Cpu)
341  {
342  // If renderModePose != RenderMode::Gpu but renderOutput, then we create an
343  // alpha = 0 pose renderer in order to keep the removing background option
344  const auto alphaKeypoint = (renderModePose != RenderMode::None
345  ? wrapperStructPose.alphaKeypoint : 0.f);
346  const auto alphaHeatMap = (renderModePose != RenderMode::None
347  ? wrapperStructPose.alphaHeatMap : 0.f);
348  // GPU rendering
349  if (renderOutputGpu)
350  {
351  for (const auto& poseExtractorNet : poseExtractorNets)
352  {
353  poseGpuRenderers.emplace_back(std::make_shared<PoseGpuRenderer>(
354  wrapperStructPose.poseModel, poseExtractorNet, wrapperStructPose.renderThreshold,
355  wrapperStructPose.blendOriginalFrame, alphaKeypoint,
356  alphaHeatMap, wrapperStructPose.defaultPartToRender
357  ));
358  }
359  }
360  // CPU rendering
361  if (renderModePose == RenderMode::Cpu)
362  {
363  poseCpuRenderer = std::make_shared<PoseCpuRenderer>(
364  wrapperStructPose.poseModel, wrapperStructPose.renderThreshold,
365  wrapperStructPose.blendOriginalFrame, alphaKeypoint, alphaHeatMap,
366  wrapperStructPose.defaultPartToRender);
367  cpuRenderers.emplace_back(std::make_shared<WPoseRenderer<TDatumsSP>>(poseCpuRenderer));
368  }
369  }
370  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
371 
372  // Pose extractor(s)
373  poseExtractorsWs.resize(poseExtractorNets.size());
374  const auto personIdExtractor = (wrapperStructExtra.identification
375  ? std::make_shared<PersonIdExtractor>() : nullptr);
376  // Keep top N people
377  // Added right after PoseExtractorNet to avoid:
378  // 1) Rendering people that are later deleted (wrong visualization).
379  // 2) Processing faces and hands on people that will be deleted (speed up).
380  // 3) Running tracking before deleting the people.
381  // Add KeepTopNPeople for each PoseExtractorNet
382  const auto keepTopNPeople = (wrapperStructPose.numberPeopleMax > 0 ?
383  std::make_shared<KeepTopNPeople>(wrapperStructPose.numberPeopleMax)
384  : nullptr);
385  // Person tracker
386  auto personTrackers = std::make_shared<std::vector<std::shared_ptr<PersonTracker>>>();
387  if (wrapperStructExtra.tracking > -1)
388  personTrackers->emplace_back(
389  std::make_shared<PersonTracker>(wrapperStructExtra.tracking == 0));
390  for (auto i = 0u; i < poseExtractorsWs.size(); i++)
391  {
392  // OpenPose keypoint detector + keepTopNPeople
393  // + ID extractor (experimental) + tracking (experimental)
394  const auto poseExtractor = std::make_shared<PoseExtractor>(
395  poseExtractorNets.at(i), keepTopNPeople, personIdExtractor, personTrackers,
396  wrapperStructPose.numberPeopleMax, wrapperStructExtra.tracking);
397  // If we want the initial image resize on GPU
398  if (cvMatToOpInputW == nullptr)
399  {
400  const auto gpuResize = true;
401  const auto cvMatToOpInput = std::make_shared<CvMatToOpInput>(
402  wrapperStructPose.poseModel, gpuResize);
403  poseExtractorsWs.at(i).emplace_back(
404  std::make_shared<WCvMatToOpInput<TDatumsSP>>(cvMatToOpInput));
405  }
406  // If we want the final image resize on GPU
407  if (addCvMatToOpOutput && cvMatToOpOutputW == nullptr)
408  {
409  const auto gpuResize = true;
410  cvMatToOpOutputs.emplace_back(std::make_shared<CvMatToOpOutput>(gpuResize));
411  poseExtractorsWs.at(i).emplace_back(
412  std::make_shared<WCvMatToOpOutput<TDatumsSP>>(cvMatToOpOutputs.back()));
413  }
414  poseExtractorsWs.at(i).emplace_back(
415  std::make_shared<WPoseExtractor<TDatumsSP>>(poseExtractor));
416  // poseExtractorsWs.at(i) = {std::make_shared<WPoseExtractor<TDatumsSP>>(poseExtractor)};
417  // // Just OpenPose keypoint detector
418  // poseExtractorsWs.at(i) = {std::make_shared<WPoseExtractorNet<TDatumsSP>>(
419  // poseExtractorNets.at(i))};
420  }
421 
422  // // (Before tracking / id extractor)
423  // // Added right after PoseExtractorNet to avoid:
424  // // 1) Rendering people that are later deleted (wrong visualization).
425  // // 2) Processing faces and hands on people that will be deleted (speed up).
426  // if (wrapperStructPose.numberPeopleMax > 0)
427  // {
428  // // Add KeepTopNPeople for each PoseExtractorNet
429  // const auto keepTopNPeople = std::make_shared<KeepTopNPeople>(
430  // wrapperStructPose.numberPeopleMax);
431  // for (auto& wPose : poseExtractorsWs)
432  // wPose.emplace_back(std::make_shared<WKeepTopNPeople<TDatumsSP>>(keepTopNPeople));
433  // }
434  }
435  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
436 
437  // Pose renderer(s)
438  if (!poseGpuRenderers.empty())
439  {
440  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
441  for (auto i = 0u; i < poseExtractorsWs.size(); i++)
442  {
443  poseExtractorsWs.at(i).emplace_back(std::make_shared<WPoseRenderer<TDatumsSP>>(
444  poseGpuRenderers.at(i)));
445  // Get shared params
446  if (!cvMatToOpOutputs.empty())
447  poseGpuRenderers.at(i)->setSharedParameters(
448  cvMatToOpOutputs.at(i)->getSharedParameters());
449  }
450  }
451  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
452 
453  // Face extractor(s)
454  if (wrapperStructFace.enable)
455  {
456  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
457  // Face detector
458  // OpenPose body-based face detector
459  if (wrapperStructFace.detector == Detector::Body)
460  {
461  // Sanity check
462  if (wrapperStructPose.poseMode == PoseMode::Disabled)
463  error("Body keypoint detection is disabled but face Detector is set to Body. Either"
464  " re-enable OpenPose body or select a different face Detector (`--face_detector`).",
465  __LINE__, __FUNCTION__, __FILE__);
466  // Constructors
467  const auto faceDetector = std::make_shared<FaceDetector>(wrapperStructPose.poseModel);
468  for (auto& wPose : poseExtractorsWs)
469  wPose.emplace_back(std::make_shared<WFaceDetector<TDatumsSP>>(faceDetector));
470  }
471  // OpenCV face detector
472  else if (wrapperStructFace.detector == Detector::OpenCV)
473  {
474  opLog("Body keypoint detection is disabled. Hence, using OpenCV face detector (much less"
475  " accurate but faster).", Priority::High);
476  for (auto& wPose : poseExtractorsWs)
477  {
478  // 1 FaceDetectorOpenCV per thread, OpenCV face detector is not thread-safe
479  const auto faceDetectorOpenCV = std::make_shared<FaceDetectorOpenCV>(modelFolder);
480  wPose.emplace_back(
481  std::make_shared<WFaceDetectorOpenCV<TDatumsSP>>(faceDetectorOpenCV)
482  );
483  }
484  }
485  // If provided by user: We do not need to create a FaceDetector
486  // Unknown face Detector
487  else if (wrapperStructFace.detector != Detector::Provided)
488  error("Unknown face Detector. Select a valid face Detector (`--face_detector`).",
489  __LINE__, __FUNCTION__, __FILE__);
490  // Face keypoint extractor
491  for (auto gpu = 0u; gpu < poseExtractorsWs.size(); gpu++)
492  {
493  // Face keypoint extractor
494  const auto netOutputSize = wrapperStructFace.netInputSize;
495  const auto faceExtractorNet = std::make_shared<FaceExtractorCaffe>(
496  wrapperStructFace.netInputSize, netOutputSize, modelFolder,
497  gpu + gpuNumberStart, wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScaleMode,
498  wrapperStructPose.enableGoogleLogging
499  );
500  faceExtractorNets.emplace_back(faceExtractorNet);
501  poseExtractorsWs.at(gpu).emplace_back(
502  std::make_shared<WFaceExtractorNet<TDatumsSP>>(faceExtractorNet));
503  }
504  }
505  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
506 
507  // Hand extractor(s)
508  if (wrapperStructHand.enable)
509  {
510  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
511  const auto handDetector = std::make_shared<HandDetector>(wrapperStructPose.poseModel);
512  for (auto gpu = 0u; gpu < poseExtractorsWs.size(); gpu++)
513  {
514  // Sanity check
515  if ((wrapperStructHand.detector == Detector::BodyWithTracking
516  || wrapperStructHand.detector == Detector::Body)
517  && wrapperStructPose.poseMode == PoseMode::Disabled)
518  error("Body keypoint detection is disabled but hand Detector is set to Body. Either"
519  " re-enable OpenPose body or select a different hand Detector (`--hand_detector`).",
520  __LINE__, __FUNCTION__, __FILE__);
521  // Hand detector
522  // OpenPose body-based hand detector with tracking
523  if (wrapperStructHand.detector == Detector::BodyWithTracking)
524  {
525  poseExtractorsWs.at(gpu).emplace_back(
526  std::make_shared<WHandDetectorTracking<TDatumsSP>>(handDetector));
527  }
528  // OpenPose body-based hand detector
529  else if (wrapperStructHand.detector == Detector::Body)
530  {
531  poseExtractorsWs.at(gpu).emplace_back(
532  std::make_shared<WHandDetector<TDatumsSP>>(handDetector));
533  }
534  // If provided by user: We do not need to create a FaceDetector
535  // Unknown hand Detector
536  else if (wrapperStructHand.detector != Detector::Provided)
537  error("Unknown hand Detector. Select a valid hand Detector (`--hand_detector`).",
538  __LINE__, __FUNCTION__, __FILE__);
539  // Hand keypoint extractor
540  const auto netOutputSize = wrapperStructHand.netInputSize;
541  const auto handExtractorNet = std::make_shared<HandExtractorCaffe>(
542  wrapperStructHand.netInputSize, netOutputSize, modelFolder,
543  gpu + gpuNumberStart, wrapperStructHand.scalesNumber, wrapperStructHand.scaleRange,
544  wrapperStructPose.heatMapTypes, wrapperStructPose.heatMapScaleMode,
545  wrapperStructPose.enableGoogleLogging
546  );
547  handExtractorNets.emplace_back(handExtractorNet);
548  poseExtractorsWs.at(gpu).emplace_back(
549  std::make_shared<WHandExtractorNet<TDatumsSP>>(handExtractorNet)
550  );
551  // If OpenPose body-based hand detector with tracking
552  if (wrapperStructHand.detector == Detector::BodyWithTracking)
553  poseExtractorsWs.at(gpu).emplace_back(
554  std::make_shared<WHandDetectorUpdate<TDatumsSP>>(handDetector));
555  }
556  }
557  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
558 
559  // Face renderer(s)
560  if (renderFace)
561  {
562  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
563  // CPU rendering
564  if (renderModeFace == RenderMode::Cpu)
565  {
566  // Construct face renderer
567  const auto faceRenderer = std::make_shared<FaceCpuRenderer>(
568  wrapperStructFace.renderThreshold, wrapperStructFace.alphaKeypoint,
569  wrapperStructFace.alphaHeatMap);
570  // Add worker
571  cpuRenderers.emplace_back(std::make_shared<WFaceRenderer<TDatumsSP>>(faceRenderer));
572  }
573  // GPU rendering
574  else if (renderModeFace == RenderMode::Gpu)
575  {
576  for (auto i = 0u; i < poseExtractorsWs.size(); i++)
577  {
578  // Construct face renderer
579  const auto faceRenderer = std::make_shared<FaceGpuRenderer>(
580  wrapperStructFace.renderThreshold, wrapperStructFace.alphaKeypoint,
581  wrapperStructFace.alphaHeatMap
582  );
583  // Performance boost -> share spGpuMemory for all renderers
584  if (!poseGpuRenderers.empty())
585  {
586  // const bool isLastRenderer = !renderHandGpu;
587  const bool isLastRenderer = !renderHandGpu && !(addCvMatToOpOutput && !addCvMatToOpOutputInCpu);
588  const auto renderer = std::static_pointer_cast<PoseGpuRenderer>(
589  poseGpuRenderers.at(i));
590  faceRenderer->setSharedParametersAndIfLast(
591  renderer->getSharedParameters(), isLastRenderer);
592  }
593  // Add worker
594  poseExtractorsWs.at(i).emplace_back(
595  std::make_shared<WFaceRenderer<TDatumsSP>>(faceRenderer));
596  }
597  }
598  else
599  error("Unknown RenderMode.", __LINE__, __FUNCTION__, __FILE__);
600  }
601  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
602 
603  // Hand renderer(s)
604  if (renderHand)
605  {
606  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
607  // CPU rendering
608  if (renderModeHand == RenderMode::Cpu)
609  {
610  // Construct hand renderer
611  const auto handRenderer = std::make_shared<HandCpuRenderer>(
612  wrapperStructHand.renderThreshold, wrapperStructHand.alphaKeypoint,
613  wrapperStructHand.alphaHeatMap);
614  // Add worker
615  cpuRenderers.emplace_back(std::make_shared<WHandRenderer<TDatumsSP>>(handRenderer));
616  }
617  // GPU rendering
618  else if (renderModeHand == RenderMode::Gpu)
619  {
620  for (auto i = 0u; i < poseExtractorsWs.size(); i++)
621  {
622  // Construct hands renderer
623  const auto handRenderer = std::make_shared<HandGpuRenderer>(
624  wrapperStructHand.renderThreshold, wrapperStructHand.alphaKeypoint,
625  wrapperStructHand.alphaHeatMap
626  );
627  // Performance boost -> share spGpuMemory for all renderers
628  if (!poseGpuRenderers.empty())
629  {
630  // const bool isLastRenderer = true;
631  const bool isLastRenderer = !(addCvMatToOpOutput && !addCvMatToOpOutputInCpu);
632  const auto renderer = std::static_pointer_cast<PoseGpuRenderer>(
633  poseGpuRenderers.at(i));
634  handRenderer->setSharedParametersAndIfLast(
635  renderer->getSharedParameters(), isLastRenderer);
636  }
637  // Add worker
638  poseExtractorsWs.at(i).emplace_back(
639  std::make_shared<WHandRenderer<TDatumsSP>>(handRenderer));
640  }
641  }
642  else
643  error("Unknown RenderMode.", __LINE__, __FUNCTION__, __FILE__);
644  }
645  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
646 
647  // Frames processor (OpenPose format -> cv::Mat format)
648  if (addCvMatToOpOutput && !addCvMatToOpOutputInCpu)
649  {
650  // for (auto& poseExtractorsW : poseExtractorsWs)
651  for (auto i = 0u ; i < poseExtractorsWs.size() ; ++i)
652  {
653  const auto gpuResize = true;
654  opOutputToCvMats.emplace_back(std::make_shared<OpOutputToCvMat>(gpuResize));
655  poseExtractorsWs.at(i).emplace_back(
656  std::make_shared<WOpOutputToCvMat<TDatumsSP>>(opOutputToCvMats.back()));
657  // Assign shared parameters
658  opOutputToCvMats.back()->setSharedParameters(
659  cvMatToOpOutputs.at(i)->getSharedParameters());
660  }
661  }
662  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
663 
664  // 3-D reconstruction
665  poseTriangulationsWs.clear();
666  if (wrapperStructExtra.reconstruct3d)
667  {
668  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
669  // For all (body/face/hands): PoseTriangulations ~30 msec, 8 GPUS ~30 msec for keypoint estimation
670  poseTriangulationsWs.resize(fastMax(1, int(poseExtractorsWs.size() / 4)));
671  for (auto i = 0u ; i < poseTriangulationsWs.size() ; i++)
672  {
673  const auto poseTriangulation = std::make_shared<PoseTriangulation>(
674  wrapperStructExtra.minViews3d);
675  poseTriangulationsWs.at(i) = {std::make_shared<WPoseTriangulation<TDatumsSP>>(
676  poseTriangulation)};
677  }
678  }
679  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
680  // Itermediate workers (e.g., OpenPose format to cv::Mat, json & frames recorder, ...)
681  postProcessingWs.clear();
682  // // Person ID identification (when no multi-thread and no dependency on tracking)
683  // if (wrapperStructExtra.identification)
684  // {
685  // const auto personIdExtractor = std::make_shared<PersonIdExtractor>();
686  // postProcessingWs.emplace_back(
687  // std::make_shared<WPersonIdExtractor<TDatumsSP>>(personIdExtractor)
688  // );
689  // }
690  // Frames processor (OpenPose format -> cv::Mat format)
691  if (addCvMatToOpOutputInCpu)
692  {
693  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
694  postProcessingWs = mergeVectors(postProcessingWs, cpuRenderers);
695  const auto opOutputToCvMat = std::make_shared<OpOutputToCvMat>();
696  postProcessingWs.emplace_back(std::make_shared<WOpOutputToCvMat<TDatumsSP>>(opOutputToCvMat));
697  }
698  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
699  // Re-scale pose if desired
700  // If desired scale is not the current input
701  if (wrapperStructPose.keypointScaleMode != ScaleMode::InputResolution
702  // and desired scale is not output when size(input) = size(output)
703  && !(wrapperStructPose.keypointScaleMode == ScaleMode::OutputResolution &&
704  (finalOutputSize == producerSize || finalOutputSize.x <= 0 || finalOutputSize.y <= 0))
705  // and desired scale is not net output when size(input) = size(net output)
706  && !(wrapperStructPose.keypointScaleMode == ScaleMode::NetOutputResolution
707  && producerSize == wrapperStructPose.netInputSize))
708  {
709  // Then we must rescale the keypoints
710  auto keypointScaler = std::make_shared<KeypointScaler>(wrapperStructPose.keypointScaleMode);
711  postProcessingWs.emplace_back(std::make_shared<WKeypointScaler<TDatumsSP>>(keypointScaler));
712  }
713  }
714  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
715 
716  // IK/Adam
717  const auto displayAdam = wrapperStructGui.displayMode == DisplayMode::DisplayAdam
718  || (wrapperStructGui.displayMode == DisplayMode::DisplayAll
719  && wrapperStructExtra.ikThreads > 0);
720  jointAngleEstimationsWs.clear();
721 #ifdef USE_3D_ADAM_MODEL
722  if (wrapperStructExtra.ikThreads > 0)
723  {
724  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
725  jointAngleEstimationsWs.resize(wrapperStructExtra.ikThreads);
726  // Pose extractor(s)
727  for (auto i = 0u; i < jointAngleEstimationsWs.size(); i++)
728  {
729  const auto jointAngleEstimation = std::make_shared<JointAngleEstimation>(displayAdam);
730  jointAngleEstimationsWs.at(i) = {std::make_shared<WJointAngleEstimation<TDatumsSP>>(
731  jointAngleEstimation)};
732  }
733  }
734  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
735 #endif
736 
737  // Output workers
738  std::vector<TWorker> outputWs;
739  // Print verbose
740  if (wrapperStructOutput.verbose > 0.)
741  {
742  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
743  const auto verbosePrinter = std::make_shared<VerbosePrinter>(
744  wrapperStructOutput.verbose, uLongLongRound(producerSharedPtr->get(getCvCapPropFrameCount())));
745  outputWs.emplace_back(std::make_shared<WVerbosePrinter<TDatumsSP>>(verbosePrinter));
746  }
747  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
748  // Send information (e.g., to Unity) though UDP client-server communication
749 
750 #ifdef USE_3D_ADAM_MODEL
751  if (!wrapperStructOutput.udpHost.empty() && !wrapperStructOutput.udpPort.empty())
752  {
753  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
754  const auto udpSender = std::make_shared<UdpSender>(wrapperStructOutput.udpHost,
755  wrapperStructOutput.udpPort);
756  outputWs.emplace_back(std::make_shared<WUdpSender<TDatumsSP>>(udpSender));
757  }
758  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
759 #endif
760  // Write people pose data on disk (json for OpenCV >= 3, xml, yml...)
761  if (!writeKeypointCleaned.empty())
762  {
763  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
764  const auto keypointSaver = std::make_shared<KeypointSaver>(writeKeypointCleaned,
765  wrapperStructOutput.writeKeypointFormat);
766  outputWs.emplace_back(std::make_shared<WPoseSaver<TDatumsSP>>(keypointSaver));
767  if (wrapperStructFace.enable)
768  outputWs.emplace_back(std::make_shared<WFaceSaver<TDatumsSP>>(keypointSaver));
769  if (wrapperStructHand.enable)
770  outputWs.emplace_back(std::make_shared<WHandSaver<TDatumsSP>>(keypointSaver));
771  }
772  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
773  // Write OpenPose output data on disk in JSON format (body/hand/face keypoints, body part locations if
774  // enabled, etc.)
775  if (!writeJsonCleaned.empty())
776  {
777  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
778  const auto peopleJsonSaver = std::make_shared<PeopleJsonSaver>(writeJsonCleaned);
779  outputWs.emplace_back(std::make_shared<WPeopleJsonSaver<TDatumsSP>>(peopleJsonSaver));
780  }
781  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
782  // Write people pose/foot/face/hand/etc. data on disk (COCO validation JSON format)
783  if (!wrapperStructOutput.writeCocoJson.empty())
784  {
785  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
786  // If humanFormat: bigger size (& maybe slower to process), but easier for user to read it
787  const auto humanFormat = true;
788  const auto cocoJsonSaver = std::make_shared<CocoJsonSaver>(
789  wrapperStructOutput.writeCocoJson.getStdString(), wrapperStructPose.poseModel, humanFormat,
790  wrapperStructOutput.writeCocoJsonVariants,
791  (wrapperStructPose.poseModel != PoseModel::CAR_22
792  && wrapperStructPose.poseModel != PoseModel::CAR_12
793  ? CocoJsonFormat::Body : CocoJsonFormat::Car),
794  wrapperStructOutput.writeCocoJsonVariant);
795  outputWs.emplace_back(std::make_shared<WCocoJsonSaver<TDatumsSP>>(cocoJsonSaver));
796  }
797  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
798  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
799  // Write frames as desired image format on hard disk
800  if (!writeImagesCleaned.empty())
801  {
802  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
803  const auto imageSaver = std::make_shared<ImageSaver>(
804  writeImagesCleaned, wrapperStructOutput.writeImagesFormat.getStdString());
805  outputWs.emplace_back(std::make_shared<WImageSaver<TDatumsSP>>(imageSaver));
806  }
807  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
808  auto originalVideoFps = 0.;
809  if (!wrapperStructOutput.writeVideo.empty() || !wrapperStructOutput.writeVideo3D.empty()
810  || !wrapperStructOutput.writeBvh.empty())
811  {
812  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
813  if (wrapperStructOutput.writeVideoFps <= 0
814  && (!oPProducer || producerSharedPtr->get(getCvCapPropFrameFps()) <= 0))
815  error("The frame rate of the frames producer is unknown. Set `--write_video_fps` to your desired"
816  " FPS if you wanna record video (`--write_video`). E.g., if it is a folder of images, you"
817  " will have to know or guess the frame rate; if it is a webcam, you should use the OpenPose"
818  " displayed FPS as desired value. If you do not care, simply add `--write_video_fps 30`.",
819  __LINE__, __FUNCTION__, __FILE__);
820  originalVideoFps = (
821  wrapperStructOutput.writeVideoFps > 0 ?
822  wrapperStructOutput.writeVideoFps : producerSharedPtr->get(getCvCapPropFrameFps()));
823  }
824  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
825  // Write frames as *.avi video on hard disk
826  if (!wrapperStructOutput.writeVideo.empty())
827  {
828  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
829  // Sanity checks
830  if (!oPProducer)
831  error("Video file can only be recorded inside `wrapper/wrapper.hpp` if the producer"
832  " is one of the default ones (e.g., video, webcam, ...).",
833  __LINE__, __FUNCTION__, __FILE__);
834  if (wrapperStructOutput.writeVideoWithAudio && producerSharedPtr->getType() != ProducerType::Video)
835  error("Audio can only be added to the output saved video if the input is also a video (either"
836  " disable `--write_video_with_audio` or use a video as input with `--video`).",
837  __LINE__, __FUNCTION__, __FILE__);
838  // Create video saver worker
839  const auto videoSaver = std::make_shared<VideoSaver>(
840  wrapperStructOutput.writeVideo.getStdString(), getCvFourcc('M','J','P','G'), originalVideoFps,
841  (wrapperStructOutput.writeVideoWithAudio ? wrapperStructInput.producerString.getStdString() : ""));
842  outputWs.emplace_back(std::make_shared<WVideoSaver<TDatumsSP>>(videoSaver));
843  }
844  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
845  // Write joint angles as *.bvh file on hard disk
846 #ifdef USE_3D_ADAM_MODEL
847  if (!wrapperStructOutput.writeBvh.empty())
848  {
849  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
850  const auto bvhSaver = std::make_shared<BvhSaver>(
851  wrapperStructOutput.writeBvh, JointAngleEstimation::getTotalModel(), originalVideoFps
852  );
853  outputWs.emplace_back(std::make_shared<WBvhSaver<TDatumsSP>>(bvhSaver));
854  }
855  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
856 #endif
857  // Write heat maps as desired image format on hard disk
858  if (!writeHeatMapsCleaned.empty())
859  {
860  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
861  const auto heatMapSaver = std::make_shared<HeatMapSaver>(
862  writeHeatMapsCleaned, wrapperStructOutput.writeHeatMapsFormat.getStdString());
863  outputWs.emplace_back(std::make_shared<WHeatMapSaver<TDatumsSP>>(heatMapSaver));
864  }
865  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
866  // Add frame information for GUI
867  const bool guiEnabled = (wrapperStructGui.displayMode != DisplayMode::NoDisplay);
868  // If this WGuiInfoAdder instance is placed before the WImageSaver or WVideoSaver, then the resulting
869  // recorded frames will look exactly as the final displayed image by the GUI
870  if (wrapperStructGui.guiVerbose && (guiEnabled || !userOutputWs.empty()
871  || threadManagerMode == ThreadManagerMode::Asynchronous
872  || threadManagerMode == ThreadManagerMode::AsynchronousOut))
873  {
874  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
875  const auto guiInfoAdder = std::make_shared<GuiInfoAdder>(numberGpuThreads, guiEnabled);
876  outputWs.emplace_back(std::make_shared<WGuiInfoAdder<TDatumsSP>>(guiInfoAdder));
877  }
878  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
879  // Minimal graphical user interface (GUI)
880  TWorker guiW;
881  TWorker videoSaver3DW;
882  if (guiEnabled)
883  {
884  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
885  // PoseRenderers to Renderers
886  std::vector<std::shared_ptr<Renderer>> renderers;
887  if (renderModePose == RenderMode::Cpu)
888  renderers.emplace_back(std::static_pointer_cast<Renderer>(poseCpuRenderer));
889  else
890  for (const auto& poseGpuRenderer : poseGpuRenderers)
891  renderers.emplace_back(std::static_pointer_cast<Renderer>(poseGpuRenderer));
892  // Display
893  const auto numberViews = (producerSharedPtr != nullptr
894  ? positiveIntRound(producerSharedPtr->get(ProducerProperty::NumberViews)) : 1);
895  auto finalOutputSizeGui = finalOutputSize;
896  if (numberViews > 1 && finalOutputSizeGui.x > 0)
897  finalOutputSizeGui.x *= numberViews;
898  // Adam (+3-D/2-D) display
899  if (displayAdam)
900  {
901 #ifdef USE_3D_ADAM_MODEL
902  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
903  // Gui
904  const auto gui = std::make_shared<GuiAdam>(
905  finalOutputSizeGui, wrapperStructGui.fullScreen, threadManager.getIsRunningSharedPtr(),
906  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers,
907  wrapperStructGui.displayMode, JointAngleEstimation::getTotalModel(),
908  wrapperStructOutput.writeVideoAdam
909  );
910  // WGui
911  guiW = {std::make_shared<WGuiAdam<TDatumsSP>>(gui)};
912  // Write 3D frames as *.avi video on hard disk
913  if (!wrapperStructOutput.writeVideo3D.empty())
914  error("3D video can only be recorded if 3D render is enabled.",
915  __LINE__, __FUNCTION__, __FILE__);
916 #endif
917  }
918  // 3-D (+2-D) display
919  else if (wrapperStructGui.displayMode == DisplayMode::Display3D
920  || wrapperStructGui.displayMode == DisplayMode::DisplayAll)
921  {
922  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
923  // Gui
924  const auto gui = std::make_shared<Gui3D>(
925  finalOutputSizeGui, wrapperStructGui.fullScreen, threadManager.getIsRunningSharedPtr(),
926  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers,
927  wrapperStructPose.poseModel, wrapperStructGui.displayMode,
928  !wrapperStructOutput.writeVideo3D.empty()
929  );
930  // WGui
931  guiW = {std::make_shared<WGui3D<TDatumsSP>>(gui)};
932  // Write 3D frames as *.avi video on hard disk
933  if (!wrapperStructOutput.writeVideo3D.empty())
934  {
935  const auto videoSaver = std::make_shared<VideoSaver>(
936  wrapperStructOutput.writeVideo3D.getStdString(), getCvFourcc('M','J','P','G'), originalVideoFps, "");
937  videoSaver3DW = std::make_shared<WVideoSaver3D<TDatumsSP>>(videoSaver);
938  }
939  }
940  // 2-D display
941  else if (wrapperStructGui.displayMode == DisplayMode::Display2D)
942  {
943  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
944  // Gui
945  const auto gui = std::make_shared<Gui>(
946  finalOutputSizeGui, wrapperStructGui.fullScreen, threadManager.getIsRunningSharedPtr(),
947  spVideoSeek, poseExtractorNets, faceExtractorNets, handExtractorNets, renderers
948  );
949  // WGui
950  guiW = {std::make_shared<WGui<TDatumsSP>>(gui)};
951  // Write 3D frames as *.avi video on hard disk
952  if (!wrapperStructOutput.writeVideo3D.empty())
953  error("3D video can only be recorded if 3D render is enabled.",
954  __LINE__, __FUNCTION__, __FILE__);
955  }
956  else
957  error("Unknown DisplayMode.", __LINE__, __FUNCTION__, __FILE__);
958  }
959  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
960  // Set FpsMax
961  TWorker wFpsMax;
962  if (wrapperStructPose.fpsMax > 0.)
963  wFpsMax = std::make_shared<WFpsMax<TDatumsSP>>(wrapperStructPose.fpsMax);
964  // Set wrapper as configured
965  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
966 
967 
968 
969 
970 
971  // The less number of queues -> the less threads opened, and potentially the less lag
972 
973  // Sanity checks
974  if ((datumProducerW == nullptr) == (userInputWs.empty())
975  && threadManagerMode != ThreadManagerMode::Asynchronous
976  && threadManagerMode != ThreadManagerMode::AsynchronousIn)
977  {
978  const auto message = "You need to have 1 and only 1 producer selected. You can introduce your own"
979  " producer by using setWorker(WorkerType::Input, ...) or use the OpenPose"
980  " default producer by configuring it in the configure function) or use the"
981  " ThreadManagerMode::Asynchronous(In) mode.";
982  error(message, __LINE__, __FUNCTION__, __FILE__);
983  }
984  if (outputWs.empty() && userOutputWs.empty() && guiW == nullptr
985  && threadManagerMode != ThreadManagerMode::Asynchronous
986  && threadManagerMode != ThreadManagerMode::AsynchronousOut)
987  {
988  error("No output selected.", __LINE__, __FUNCTION__, __FILE__);
989  }
990 
991  // Thread Manager
992  // Clean previous thread manager (avoid configure to crash the program if used more than once)
993  threadManager.reset();
994  unsigned long long threadId = 0ull;
995  auto queueIn = 0ull;
996  auto queueOut = 1ull;
997  // After producer
998  // ID generator (before any multi-threading or any function that requires the ID)
999  const auto wIdGenerator = std::make_shared<WIdGenerator<TDatumsSP>>();
1000  // If custom user Worker and uses its own thread
1001  std::vector<TWorker> workersAux;
1002  if (!userPreProcessingWs.empty())
1003  {
1004  // If custom user Worker in its own thread
1005  if (userPreProcessingWsOnNewThread)
1006  opLog("You chose to add your pre-processing function in a new thread. However, OpenPose will"
1007  " add it in the same thread than the input frame producer.",
1008  Priority::High, __LINE__, __FUNCTION__, __FILE__);
1009  workersAux = mergeVectors(workersAux, {userPreProcessingWs});
1010  }
1011  workersAux = mergeVectors(workersAux, {wIdGenerator});
1012  // Scale & cv::Mat to OP format
1013  if (scaleAndSizeExtractorW != nullptr)
1014  workersAux = mergeVectors(workersAux, {scaleAndSizeExtractorW});
1015  if (cvMatToOpInputW != nullptr)
1016  workersAux = mergeVectors(workersAux, {cvMatToOpInputW});
1017  // cv::Mat to output format
1018  if (cvMatToOpOutputW != nullptr)
1019  workersAux = mergeVectors(workersAux, {cvMatToOpOutputW});
1020 
1021  // Producer
1022  // If custom user Worker and uses its own thread
1023  if (!userInputWs.empty() && userInputWsOnNewThread)
1024  {
1025  // Thread 0, queues 0 -> 1
1026  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1027  threadManager.add(threadId, userInputWs, queueIn++, queueOut++);
1028  threadIdPP(threadId, multiThreadEnabled);
1029  }
1030  // If custom user Worker in same thread
1031  else if (!userInputWs.empty())
1032  workersAux = mergeVectors(userInputWs, workersAux);
1033  // If OpenPose producer (same thread)
1034  else if (datumProducerW != nullptr)
1035  workersAux = mergeVectors({datumProducerW}, workersAux);
1036  // Otherwise
1037  else if (threadManagerMode != ThreadManagerMode::Asynchronous
1038  && threadManagerMode != ThreadManagerMode::AsynchronousIn)
1039  error("No input selected.", __LINE__, __FUNCTION__, __FILE__);
1040  // Thread 0 or 1, queues 0 -> 1
1041  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1042  threadManager.add(threadId, workersAux, queueIn++, queueOut++);
1043  // Increase thread
1044  threadIdPP(threadId, multiThreadEnabled);
1045 
1046  // Pose estimation & rendering
1047  // Thread 1 or 2...X, queues 1 -> 2, X = 2 + #GPUs
1048  if (!poseExtractorsWs.empty())
1049  {
1050  if (multiThreadEnabled)
1051  {
1052  for (auto& wPose : poseExtractorsWs)
1053  {
1054  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1055  threadManager.add(threadId, wPose, queueIn, queueOut);
1056  threadIdPP(threadId, multiThreadEnabled);
1057  }
1058  queueIn++;
1059  queueOut++;
1060  // Sort frames - Required own thread
1061  if (poseExtractorsWs.size() > 1u)
1062  {
1063  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1064  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1065  threadManager.add(threadId, wQueueOrderer, queueIn++, queueOut++);
1066  threadIdPP(threadId, multiThreadEnabled);
1067  }
1068  }
1069  else
1070  {
1071  if (poseExtractorsWs.size() > 1)
1072  opLog("Multi-threading disabled, only 1 thread running. All GPUs have been disabled but the"
1073  " first one, which is defined by gpuNumberStart (e.g., in the OpenPose demo, it is set"
1074  " with the `--num_gpu_start` flag).", Priority::High);
1075  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1076  threadManager.add(threadId, poseExtractorsWs.at(0), queueIn++, queueOut++);
1077  }
1078  }
1079  // Assemble all frames from same time instant (3-D module)
1080  const auto wQueueAssembler = std::make_shared<WQueueAssembler<TDatums>>();
1081  // 3-D reconstruction
1082  if (!poseTriangulationsWs.empty())
1083  {
1084  // Assemble frames
1085  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1086  threadManager.add(threadId, wQueueAssembler, queueIn++, queueOut++);
1087  threadIdPP(threadId, multiThreadEnabled);
1088  // 3-D reconstruction
1089  if (multiThreadEnabled)
1090  {
1091  for (auto& wPoseTriangulations : poseTriangulationsWs)
1092  {
1093  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1094  threadManager.add(threadId, wPoseTriangulations, queueIn, queueOut);
1095  threadIdPP(threadId, multiThreadEnabled);
1096  }
1097  queueIn++;
1098  queueOut++;
1099  // Sort frames
1100  if (poseTriangulationsWs.size() > 1u)
1101  {
1102  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1103  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1104  threadManager.add(threadId, wQueueOrderer, queueIn++, queueOut++);
1105  threadIdPP(threadId, multiThreadEnabled);
1106  }
1107  }
1108  else
1109  {
1110  if (poseTriangulationsWs.size() > 1)
1111  opLog("Multi-threading disabled, only 1 thread running for 3-D triangulation.",
1112  Priority::High);
1113  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1114  threadManager.add(threadId, poseTriangulationsWs.at(0), queueIn++, queueOut++);
1115  }
1116  }
1117  else
1118  postProcessingWs = mergeVectors({wQueueAssembler}, postProcessingWs);
1119  // Adam/IK step
1120  if (!jointAngleEstimationsWs.empty())
1121  {
1122  if (multiThreadEnabled)
1123  {
1124  for (auto& wJointAngleEstimator : jointAngleEstimationsWs)
1125  {
1126  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1127  threadManager.add(threadId, wJointAngleEstimator, queueIn, queueOut);
1128  threadIdPP(threadId, multiThreadEnabled);
1129  }
1130  queueIn++;
1131  queueOut++;
1132  // Sort frames
1133  if (jointAngleEstimationsWs.size() > 1)
1134  {
1135  const auto wQueueOrderer = std::make_shared<WQueueOrderer<TDatumsSP>>();
1136  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1137  threadManager.add(threadId, wQueueOrderer, queueIn++, queueOut++);
1138  threadIdPP(threadId, multiThreadEnabled);
1139  }
1140  }
1141  else
1142  {
1143  if (jointAngleEstimationsWs.size() > 1)
1144  opLog("Multi-threading disabled, only 1 thread running for joint angle estimation.",
1145  Priority::High);
1146  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1147  threadManager.add(threadId, jointAngleEstimationsWs.at(0), queueIn++, queueOut++);
1148  }
1149  }
1150  // Post processing workers
1151  if (!postProcessingWs.empty())
1152  {
1153  // Combining postProcessingWs and outputWs
1154  outputWs = mergeVectors(postProcessingWs, outputWs);
1155  // // If I wanna split them
1156  // opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1157  // threadManager.add(threadId, postProcessingWs, queueIn++, queueOut++);
1158  // threadIdPP(threadId, multiThreadEnabled);
1159  }
1160  // If custom user Worker and uses its own thread
1161  if (!userPostProcessingWs.empty())
1162  {
1163  // If custom user Worker in its own thread
1164  if (userPostProcessingWsOnNewThread)
1165  {
1166  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1167  threadManager.add(threadId, userPostProcessingWs, queueIn++, queueOut++);
1168  threadIdPP(threadId, multiThreadEnabled);
1169  }
1170  // If custom user Worker in same thread
1171  // Merge with outputWs
1172  else
1173  outputWs = mergeVectors(outputWs, userPostProcessingWs);
1174  }
1175  // Output workers
1176  if (!outputWs.empty())
1177  {
1178  // Thread 4 or 5, queues 4 -> 5
1179  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1180  threadManager.add(threadId, outputWs, queueIn++, queueOut++);
1181  threadIdPP(threadId, multiThreadEnabled);
1182  }
1183  // User output worker
1184  // Thread Y, queues Q -> Q+1
1185  if (!userOutputWs.empty())
1186  {
1187  if (userOutputWsOnNewThread)
1188  {
1189  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1190  threadManager.add(threadId, userOutputWs, queueIn++, queueOut++);
1191  threadIdPP(threadId, multiThreadEnabled);
1192  }
1193  else
1194  {
1195  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1196  threadManager.add(threadId-1, userOutputWs, queueIn++, queueOut++);
1197  }
1198  }
1199  // OpenPose GUI
1200  if (guiW != nullptr)
1201  {
1202  // Thread Y+1, queues Q+1 -> Q+2
1203  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1204  threadManager.add(threadId, guiW, queueIn++, queueOut++);
1205  // Saving 3D output
1206  if (videoSaver3DW != nullptr)
1207  threadManager.add(threadId, videoSaver3DW, queueIn++, queueOut++);
1208  threadIdPP(threadId, multiThreadEnabled);
1209  }
1210  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1211  // Setting maximum speed
1212  if (wFpsMax != nullptr)
1213  {
1214  opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__);
1215  threadManager.add(threadId, wFpsMax, queueIn++, queueOut++);
1216  threadIdPP(threadId, multiThreadEnabled);
1217  }
1218  }
1219  catch (const std::exception& e)
1220  {
1221  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1222  }
1223  }
1224 
1225  template<typename TDatum, typename TDatums, typename TDatumsSP>
1226  void createMultiviewTDatum(
1227  TDatumsSP& tDatumsSP, unsigned long long& frameCounter,
1228  const CameraParameterReader& cameraParameterReader, const void* const cvMatPtr)
1229  {
1230  try
1231  {
1232  // Sanity check
1233  if (tDatumsSP == nullptr)
1234  op::error("tDatumsSP was nullptr, it must be initialized.", __LINE__, __FUNCTION__, __FILE__);
1235  // Camera parameters
1236  const std::vector<op::Matrix>& cameraMatrices = cameraParameterReader.getCameraMatrices();
1237  const std::vector<op::Matrix>& cameraIntrinsics = cameraParameterReader.getCameraIntrinsics();
1238  const std::vector<op::Matrix>& cameraExtrinsics = cameraParameterReader.getCameraExtrinsics();
1239  const auto matrixesSize = cameraMatrices.size();
1240  // More sanity checks
1241  if (cameraMatrices.size() < 2)
1242  op::error("There is less than 2 camera parameter matrices.",
1243  __LINE__, __FUNCTION__, __FILE__);
1244  if (cameraMatrices.size() != cameraIntrinsics.size() || cameraMatrices.size() != cameraExtrinsics.size())
1245  op::error("Camera parameters must have the same size.", __LINE__, __FUNCTION__, __FILE__);
1246  // Split image to process
1247  std::vector<op::Matrix> imagesToProcess(matrixesSize);
1248  op::Matrix::splitCvMatIntoVectorMatrix(imagesToProcess, cvMatPtr);
1249  // Fill tDatumsSP
1250  tDatumsSP->resize(cameraMatrices.size());
1251  for (auto datumIndex = 0 ; datumIndex < matrixesSize ; ++datumIndex)
1252  {
1253  auto& datumPtr = tDatumsSP->at(datumIndex);
1254  datumPtr = std::make_shared<op::Datum>();
1255  datumPtr->frameNumber = frameCounter;
1256  datumPtr->cvInputData = imagesToProcess[datumIndex];
1257  if (matrixesSize > 1)
1258  {
1259  datumPtr->subId = datumIndex;
1260  datumPtr->subIdMax = matrixesSize-1;
1261  datumPtr->cameraMatrix = cameraMatrices[datumIndex];
1262  datumPtr->cameraExtrinsics = cameraExtrinsics[datumIndex];
1263  datumPtr->cameraIntrinsics = cameraIntrinsics[datumIndex];
1264  }
1265  }
1266  ++frameCounter;
1267  }
1268  catch (const std::exception& e)
1269  {
1270  error(e.what(), __LINE__, __FUNCTION__, __FILE__);
1271  }
1272  }
1273 }
1274 
1275 #endif // OPENPOSE_WRAPPER_WRAPPER_AUXILIARY_HPP
op::CameraParameterReader::getCameraIntrinsics
const std::vector< Matrix > & getCameraIntrinsics() const
op::CameraParameterReader::getCameraExtrinsics
const std::vector< Matrix > & getCameraExtrinsics() const
op::CameraParameterReader::getCameraMatrices
const std::vector< Matrix > & getCameraMatrices() const
op::Matrix::splitCvMatIntoVectorMatrix
static void splitCvMatIntoVectorMatrix(std::vector< Matrix > &matrixesResized, const void *const cvMatPtr)
op::String::getStdString
const std::string & getStdString() const
op::String::empty
bool empty() const
op::ThreadManager::getIsRunningSharedPtr
std::shared_ptr< std::atomic< bool > > getIsRunningSharedPtr()
Definition: threadManager.hpp:49
op::ThreadManager::add
void add(const unsigned long long threadId, const std::vector< TWorker > &tWorkers, const unsigned long long queueInId, const unsigned long long queueOutId)
Definition: threadManager.hpp:134
OP_API
#define OP_API
Definition: macros.hpp:18
op::getCvCapPropFrameCount
OP_API int getCvCapPropFrameCount()
op::getCvCapPropFrameHeight
OP_API int getCvCapPropFrameHeight()
op::ThreadManagerMode
ThreadManagerMode
Definition: enumClasses.hpp:10
op::createMultiviewTDatum
void createMultiviewTDatum(TDatumsSP &tDatumsSP, unsigned long long &frameCounter, const CameraParameterReader &cameraParameterReader, const void *const cvMatPtr)
Definition: wrapperAuxiliary.hpp:1226
op::getCvFourcc
OP_API int getCvFourcc(const char c1, const char c2, const char c3, const char c4)
op::configureThreadManager
void configureThreadManager(ThreadManager< TDatumsSP > &threadManager, const bool multiThreadEnabled, const ThreadManagerMode threadManagerMode, const WrapperStructPose &wrapperStructPose, const WrapperStructFace &wrapperStructFace, const WrapperStructHand &wrapperStructHand, const WrapperStructExtra &wrapperStructExtra, const WrapperStructInput &wrapperStructInput, const WrapperStructOutput &wrapperStructOutput, const WrapperStructGui &wrapperStructGui, const std::array< std::vector< TWorker >, int(WorkerType::Size)> &userWs, const std::array< bool, int(WorkerType::Size)> &userWsOnNewThread)
Definition: wrapperAuxiliary.hpp:98
op::error
OP_API void error(const std::string &message, const int line=-1, const std::string &function="", const std::string &file="")
op::positiveIntRound
int positiveIntRound(const T a)
Definition: fastMath.hpp:29
op::createProducer
OP_API std::shared_ptr< Producer > createProducer(const ProducerType producerType=ProducerType::None, const std::string &producerString="", const Point< int > &cameraResolution=Point< int >{-1,-1}, const std::string &cameraParameterPath="models/cameraParameters/", const bool undistortImage=true, const int numberViews=-1)
op::getCvCapPropFrameWidth
OP_API int getCvCapPropFrameWidth()
op::uLongLongRound
unsigned long long uLongLongRound(const T a)
Definition: fastMath.hpp:66
op::getGpuMode
OP_API GpuMode getGpuMode()
op::fastMax
T fastMax(const T a, const T b)
Definition: fastMath.hpp:73
op::mergeVectors
std::vector< T > mergeVectors(const std::vector< T > &vectorA, const std::vector< T > &vectorB)
Definition: standard.hpp:40
op::opLog
OP_API void opLog(const std::string &message, const Priority priority=Priority::Max, const int line=-1, const std::string &function="", const std::string &file="")
op::getGpuNumber
OP_API int getGpuNumber()
op::getCvCapPropFrameFps
OP_API int getCvCapPropFrameFps()
op::formatAsDirectory
OP_API std::string formatAsDirectory(const std::string &directoryPathString)
op::wrapperConfigureSanityChecks
OP_API void wrapperConfigureSanityChecks(WrapperStructPose &wrapperStructPose, const WrapperStructFace &wrapperStructFace, const WrapperStructHand &wrapperStructHand, const WrapperStructExtra &wrapperStructExtra, const WrapperStructInput &wrapperStructInput, const WrapperStructOutput &wrapperStructOutput, const WrapperStructGui &wrapperStructGui, const bool renderOutput, const bool userInputAndPreprocessingWsEmpty, const bool userOutputWsEmpty, const std::shared_ptr< Producer > &producerSharedPtr, const ThreadManagerMode threadManagerMode)
op::threadIdPP
OP_API void threadIdPP(unsigned long long &threadId, const bool multiThreadEnabled)
op::WrapperStructInput::frameLast
unsigned long long frameLast
Definition: wrapperStructInput.hpp:46
op::WrapperStructInput::frameStep
unsigned long long frameStep
Definition: wrapperStructInput.hpp:40
op::WrapperStructInput::frameFirst
unsigned long long frameFirst
Definition: wrapperStructInput.hpp:33
op::WrapperStructPose::heatMapTypes
std::vector< HeatMapType > heatMapTypes
Definition: wrapperStructPose.hpp:137