async_pipeline.h

/*
// Copyright (C) 2020-2024 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/
 
#pragma once
#include <stdint.h>
 
#include <condition_variable>
#include <exception>
#include <memory>
#include <mutex>
#include <unordered_map>
 
#include <openvino/openvino.hpp>
 
#include <models/results.h>
#include <utils/performance_metrics.hpp>
 
#include "pipelines/requests_pool.h"
 
class ModelBase;
struct InputData;
struct MetaData;
struct ModelConfig;
 
class AsyncPipeline {
public:
    AsyncPipeline(std::unique_ptr<ModelBase>&& modelInstance, const ModelConfig& config, ov::Core& core);
    virtual ~AsyncPipeline();
 
    void waitForData(bool shouldKeepOrder = true);
 
    bool isReadyToProcess() {
        return requestsPool->isIdleRequestAvailable();
    }
 
    void waitForTotalCompletion() {
        if (requestsPool)
            requestsPool->waitForTotalCompletion();
    }
 
    virtual int64_t submitData(const InputData& inputData, const std::shared_ptr<MetaData>& metaData);
 
    virtual std::unique_ptr<ResultBase> getResult(bool shouldKeepOrder = true);
 
    PerformanceMetrics getInferenceMetircs() {
        return inferenceMetrics;
    }
    PerformanceMetrics getPreprocessMetrics() {
        return preprocessMetrics;
    }
    PerformanceMetrics getPostprocessMetrics() {
        return postprocessMetrics;
    }
 
protected:
    virtual InferenceResult getInferenceResult(bool shouldKeepOrder);
 
    std::unique_ptr<RequestsPool> requestsPool;
    std::unordered_map<int64_t, InferenceResult> completedInferenceResults;
 
    ov::CompiledModel compiledModel;
 
    std::mutex mtx;
    std::condition_variable condVar;
 
    int64_t inputFrameId = 0;
    int64_t outputFrameId = 0;
 
    std::exception_ptr callbackException = nullptr;
 
    std::unique_ptr<ModelBase> model;
    PerformanceMetrics inferenceMetrics;
    PerformanceMetrics preprocessMetrics;
    PerformanceMetrics postprocessMetrics;
};