Direct3DInterop.cpp

#include "pch.h"
#include "Direct3DInterop.h"
#include "Direct3DContentProvider.h"
#include <windows.storage.streams.h>
#include <wrl.h>
#include <robuffer.h>
#include <opencv2\core.hpp>
#include <opencv2\imgproc.hpp>
#include <opencv2\features2d.hpp>
#include <algorithm>

using namespace Windows::Storage::Streams;
using namespace Microsoft::WRL;
using namespace Windows::Foundation;
using namespace Windows::UI::Core;
using namespace Microsoft::WRL;
using namespace Windows::Phone::Graphics::Interop;
using namespace Windows::Phone::Input::Interop;
using namespace Windows::Foundation;
using namespace Windows::Foundation::Collections;
using namespace Windows::Phone::Media::Capture;

#if !defined(_M_ARM)
#pragma message("warning: Direct3DInterop.cpp: Windows Phone camera code does not run in the emulator.")
#pragma message("warning: Direct3DInterop.cpp: Please compile as an ARM build and run on a device.")
#endif

namespace PhoneXamlDirect3DApp1Comp
{
    // Called each time a preview frame is available
    void CameraCapturePreviewSink::OnFrameAvailable(
        DXGI_FORMAT format,
        UINT width,
        UINT height,
        BYTE* pixels
        )
    {
        m_Direct3dInterop->UpdateFrame(pixels, width, height);
    }

    // Called each time a captured frame is available
    void CameraCaptureSampleSink::OnSampleAvailable(
        ULONGLONG hnsPresentationTime,
        ULONGLONG hnsSampleDuration,
        DWORD cbSample,
        BYTE* pSample)
    {


    }

    Direct3DInterop::Direct3DInterop()
        : m_algorithm(OCVFilterType::ePreview)
        , m_contentDirty(false)
        , m_backFrame(nullptr)
        , m_frontFrame(nullptr)
    {
    }

    bool Direct3DInterop::SwapFrames()
    {
        std::lock_guard<std::mutex> lock(m_mutex);
        if(m_backFrame != nullptr)
        {
            std::swap(m_backFrame, m_frontFrame);
            return true;
        }
        return false;
    }

    void Direct3DInterop::UpdateFrame(byte* buffer,int width,int height)
    {
        std::lock_guard<std::mutex> lock(m_mutex);
        if(m_backFrame == nullptr)
        {
            m_backFrame = std::shared_ptr<cv::Mat> (new cv::Mat(height, width, CV_8UC4));
            m_frontFrame = std::shared_ptr<cv::Mat> (new cv::Mat(height, width, CV_8UC4));
        }

        memcpy(m_backFrame.get()->data, buffer, 4 * height*width);
        m_contentDirty = true;
        RequestAdditionalFrame();
    }

    void Direct3DInterop::ProcessFrame()
    {
        if (SwapFrames())
        {
            if (m_renderer)
            {
                cv::Mat* mat = m_frontFrame.get();

                switch (m_algorithm)
                {
                    case OCVFilterType::ePreview:
                    {
                        break;
                    }

                    case OCVFilterType::eGray:
                    {
                        ApplyGrayFilter(mat);
                        break;
                    }

                    case OCVFilterType::eCanny:
                    {
                        ApplyCannyFilter(mat);
                        break;
                    }

                    case OCVFilterType::eBlur:
                    {
                        ApplyBlurFilter(mat);
                        break;
                    }

                    case OCVFilterType::eFindFeatures:
                    {
                        ApplyFindFeaturesFilter(mat);
                        break;
                    }

                    case OCVFilterType::eSepia:
                    {
                        ApplySepiaFilter(mat);
                        break;
                    }
                }

                m_renderer->CreateTextureFromByte(mat->data, mat->cols, mat->rows);
            }
        }
    }

    void Direct3DInterop::ApplyGrayFilter(cv::Mat* mat)
    {
        cv::Mat intermediateMat;
        cv::cvtColor(*mat, intermediateMat, COLOR_RGBA2GRAY);
        cv::cvtColor(intermediateMat, *mat, COLOR_GRAY2BGRA);
    }

    void Direct3DInterop::ApplyCannyFilter(cv::Mat* mat)
    {
        cv::Mat intermediateMat;
        cv::Canny(*mat, intermediateMat, 80, 90);
        cv::cvtColor(intermediateMat, *mat, COLOR_GRAY2BGRA);
    }

    void Direct3DInterop::ApplyBlurFilter(cv::Mat* mat)
    {
        cv::Mat intermediateMat;
        //	cv::Blur(image, intermediateMat, 80, 90);
        cv::cvtColor(intermediateMat, *mat, COLOR_GRAY2BGRA);
    }

    void Direct3DInterop::ApplyFindFeaturesFilter(cv::Mat* mat)
    {
        cv::Mat intermediateMat;
        cv::Ptr<cv::FastFeatureDetector> detector = cv::FastFeatureDetector::create(50);
        std::vector<cv::KeyPoint> features;

        cv::cvtColor(*mat, intermediateMat, COLOR_RGBA2GRAY);
        detector->detect(intermediateMat, features);

        for( unsigned int i = 0; i < std::min(features.size(), (size_t)50); i++ )
        {
            const cv::KeyPoint& kp = features[i];
            cv::circle(*mat, cv::Point((int)kp.pt.x, (int)kp.pt.y), 10, cv::Scalar(255,0,0,255));
        }
    }

    void Direct3DInterop::ApplySepiaFilter(cv::Mat* mat)
    {
        const float SepiaKernelData[16] =
        {
            /* B */0.131f, 0.534f, 0.272f, 0.f,
            /* G */0.168f, 0.686f, 0.349f, 0.f,
            /* R */0.189f, 0.769f, 0.393f, 0.f,
            /* A */0.000f, 0.000f, 0.000f, 1.f
        };

        const cv::Mat SepiaKernel(4, 4, CV_32FC1, (void*)SepiaKernelData);
        cv::transform(*mat, *mat, SepiaKernel);
    }

    IDrawingSurfaceContentProvider^ Direct3DInterop::CreateContentProvider()
    {
        ComPtr<Direct3DContentProvider> provider = Make<Direct3DContentProvider>(this);
        return reinterpret_cast<IDrawingSurfaceContentProvider^>(provider.Detach());
    }

    // IDrawingSurfaceManipulationHandler
    void Direct3DInterop::SetManipulationHost(DrawingSurfaceManipulationHost^ manipulationHost)
    {
        manipulationHost->PointerPressed +=
            ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerPressed);

        manipulationHost->PointerMoved +=
            ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerMoved);

        manipulationHost->PointerReleased +=
            ref new TypedEventHandler<DrawingSurfaceManipulationHost^, PointerEventArgs^>(this, &Direct3DInterop::OnPointerReleased);
    }

    void Direct3DInterop::RenderResolution::set(Windows::Foundation::Size renderResolution)
    {
        if (renderResolution.Width  != m_renderResolution.Width ||
            renderResolution.Height != m_renderResolution.Height)
        {
            m_renderResolution = renderResolution;

            if (m_renderer)
            {
                m_renderer->UpdateForRenderResolutionChange(m_renderResolution.Width, m_renderResolution.Height);
                RecreateSynchronizedTexture();
            }
        }
    }

    // Event Handlers

    void Direct3DInterop::OnPointerPressed(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
    {
        // Insert your code here.
    }

    void Direct3DInterop::OnPointerMoved(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
    {
        // Insert your code here.
    }

    void Direct3DInterop::OnPointerReleased(DrawingSurfaceManipulationHost^ sender, PointerEventArgs^ args)
    {
        // Insert your code here.
    }

    void Direct3DInterop::StartCamera()
    {
        // Set the capture dimensions
        Size captureDimensions;
        captureDimensions.Width = 640;
        captureDimensions.Height = 480;

        // Open the AudioVideoCaptureDevice for video only
        IAsyncOperation<AudioVideoCaptureDevice^> ^openOperation = AudioVideoCaptureDevice::OpenForVideoOnlyAsync(CameraSensorLocation::Back, captureDimensions);

        openOperation->Completed = ref new AsyncOperationCompletedHandler<AudioVideoCaptureDevice^>(
            [this] (IAsyncOperation<AudioVideoCaptureDevice^> ^operation, Windows::Foundation::AsyncStatus status)
            {
                if (status == Windows::Foundation::AsyncStatus::Completed)
                {
                    auto captureDevice = operation->GetResults();

                    // Save the reference to the opened video capture device
                    pAudioVideoCaptureDevice = captureDevice;

                    // Retrieve the native ICameraCaptureDeviceNative interface from the managed video capture device
                    ICameraCaptureDeviceNative *iCameraCaptureDeviceNative = NULL;
                    HRESULT hr = reinterpret_cast<IUnknown*>(captureDevice)->QueryInterface(__uuidof(ICameraCaptureDeviceNative), (void**) &iCameraCaptureDeviceNative);

                    // Save the pointer to the native interface
                    pCameraCaptureDeviceNative = iCameraCaptureDeviceNative;

                    // Initialize the preview dimensions (see the accompanying article at )
                    // The aspect ratio of the capture and preview resolution must be equal,
                    // 4:3 for capture => 4:3 for preview, and 16:9 for capture => 16:9 for preview.
                    Size previewDimensions;
                    previewDimensions.Width = 640;
                    previewDimensions.Height = 480;

                    IAsyncAction^ setPreviewResolutionAction = pAudioVideoCaptureDevice->SetPreviewResolutionAsync(previewDimensions);
                    setPreviewResolutionAction->Completed = ref new AsyncActionCompletedHandler(
                        [this](IAsyncAction^ action, Windows::Foundation::AsyncStatus status)
                        {
                            HResult hr = action->ErrorCode;

                            if (status == Windows::Foundation::AsyncStatus::Completed)
                            {
                                // Create the sink
                                MakeAndInitialize<CameraCapturePreviewSink>(&pCameraCapturePreviewSink);
                                pCameraCapturePreviewSink->SetDelegate(this);
                                pCameraCaptureDeviceNative->SetPreviewSink(pCameraCapturePreviewSink);

                                // Set the preview format
                                pCameraCaptureDeviceNative->SetPreviewFormat(DXGI_FORMAT::DXGI_FORMAT_B8G8R8A8_UNORM);
                            }
                        }
                    );

                    // Retrieve IAudioVideoCaptureDeviceNative native interface from managed projection.
                    IAudioVideoCaptureDeviceNative *iAudioVideoCaptureDeviceNative = NULL;
                    hr = reinterpret_cast<IUnknown*>(captureDevice)->QueryInterface(__uuidof(IAudioVideoCaptureDeviceNative), (void**) &iAudioVideoCaptureDeviceNative);

                    // Save the pointer to the IAudioVideoCaptureDeviceNative native interface
                    pAudioVideoCaptureDeviceNative = iAudioVideoCaptureDeviceNative;

                    // Set sample encoding format to ARGB. See the documentation for further values.
                    pAudioVideoCaptureDevice->VideoEncodingFormat = CameraCaptureVideoFormat::Argb;

                    // Initialize and set the CameraCaptureSampleSink class as sink for captures samples
                    MakeAndInitialize<CameraCaptureSampleSink>(&pCameraCaptureSampleSink);
                    pAudioVideoCaptureDeviceNative->SetVideoSampleSink(pCameraCaptureSampleSink);

                    // Start recording (only way to receive samples using the ICameraCaptureSampleSink interface
                    pAudioVideoCaptureDevice->StartRecordingToSinkAsync();
                }
            }
        );

    }
    // Interface With Direct3DContentProvider
    HRESULT Direct3DInterop::Connect(_In_ IDrawingSurfaceRuntimeHostNative* host)
    {
        m_renderer = ref new QuadRenderer();
        m_renderer->Initialize();
        m_renderer->UpdateForWindowSizeChange(WindowBounds.Width, WindowBounds.Height);
        m_renderer->UpdateForRenderResolutionChange(m_renderResolution.Width, m_renderResolution.Height);
        StartCamera();

        return S_OK;
    }

    void Direct3DInterop::Disconnect()
    {
        m_renderer = nullptr;
    }

    HRESULT Direct3DInterop::PrepareResources(_In_ const LARGE_INTEGER* presentTargetTime, _Out_ BOOL* contentDirty)
    {
        *contentDirty = m_contentDirty;
        if(m_contentDirty)
        {
            ProcessFrame();
        }
        m_contentDirty = false;
        return S_OK;
    }

    HRESULT Direct3DInterop::GetTexture(_In_ const DrawingSurfaceSizeF* size, _Out_ IDrawingSurfaceSynchronizedTextureNative** synchronizedTexture, _Out_ DrawingSurfaceRectF* textureSubRectangle)
    {
        m_renderer->Update();
        m_renderer->Render();
        return S_OK;
    }

    ID3D11Texture2D* Direct3DInterop::GetTexture()
    {
        return m_renderer->GetTexture();
    }
}