CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
CoCalc provides the best real-time collaborative environment for Jupyter Notebooks, LaTeX documents, and SageMath, scalable from individual users to large groups and classes!
Path: blob/master/Windows/CaptureDevice.cpp
Views: 1401
// Copyright (c) 2020- PPSSPP Project.12// This program is free software: you can redistribute it and/or modify3// it under the terms of the GNU General Public License as published by4// the Free Software Foundation, version 2.0 or later versions.56// This program is distributed in the hope that it will be useful,7// but WITHOUT ANY WARRANTY; without even the implied warranty of8// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the9// GNU General Public License 2.0 for more details.1011// A copy of the GPL 2.0 should have been included with the program.12// If not, see http://www.gnu.org/licenses/1314// Official git repository and contact information can be found at15// https://github.com/hrydgard/ppsspp and http://www.ppsspp.org/.1617#include <shlwapi.h>1819#include "Common/Thread/ThreadUtil.h"20#include "CaptureDevice.h"21#include "BufferLock.h"22#include "ext/jpge/jpge.h"23#include "CommonTypes.h"24#include "Core/HLE/sceUsbCam.h"25#include "Core/Config.h"2627namespace MFAPI {28HINSTANCE Mflib;29HINSTANCE Mfplatlib;30HINSTANCE Mfreadwritelib;3132typedef HRESULT(WINAPI *MFEnumDeviceSourcesFunc)(IMFAttributes *, IMFActivate ***, UINT32 *);33typedef HRESULT(WINAPI *MFGetStrideForBitmapInfoHeaderFunc)(DWORD, DWORD, LONG *);34typedef HRESULT(WINAPI *MFCreateSourceReaderFromMediaSourceFunc)(IMFMediaSource *, IMFAttributes *, IMFSourceReader **);35typedef HRESULT(WINAPI *MFCopyImageFunc)(BYTE *, LONG, const BYTE *, LONG, DWORD, DWORD);3637MFEnumDeviceSourcesFunc EnumDeviceSources;38MFGetStrideForBitmapInfoHeaderFunc GetStrideForBitmapInfoHeader;39MFCreateSourceReaderFromMediaSourceFunc CreateSourceReaderFromMediaSource;40MFCopyImageFunc CopyImage;41}4243using namespace MFAPI;4445bool RegisterCMPTMFApis(){46//For the compatibility,these funcs don't be supported on vista.47Mflib = LoadLibrary(L"Mf.dll");48Mfplatlib = LoadLibrary(L"Mfplat.dll");49Mfreadwritelib = LoadLibrary(L"Mfreadwrite.dll");50if (!Mflib || !Mfplatlib || !Mfreadwritelib)51return false;5253EnumDeviceSources = (MFEnumDeviceSourcesFunc)GetProcAddress(Mflib, "MFEnumDeviceSources");54GetStrideForBitmapInfoHeader = (MFGetStrideForBitmapInfoHeaderFunc)GetProcAddress(Mfplatlib, "MFGetStrideForBitmapInfoHeader");55MFAPI::CopyImage = (MFCopyImageFunc)GetProcAddress(Mfplatlib, "MFCopyImage");56CreateSourceReaderFromMediaSource = (MFCreateSourceReaderFromMediaSourceFunc)GetProcAddress(Mfreadwritelib, "MFCreateSourceReaderFromMediaSource");57if (!EnumDeviceSources || !GetStrideForBitmapInfoHeader || !CreateSourceReaderFromMediaSource || !MFAPI::CopyImage)58return false;5960return true;61}6263bool unRegisterCMPTMFApis() {64if (Mflib) {65FreeLibrary(Mflib);66Mflib = nullptr;67}6869if (Mfplatlib) {70FreeLibrary(Mfplatlib);71Mfplatlib = nullptr;72}7374if (Mfreadwritelib) {75FreeLibrary(Mfreadwritelib);76Mfreadwritelib = nullptr;77}7879EnumDeviceSources = nullptr;80GetStrideForBitmapInfoHeader = nullptr;81CreateSourceReaderFromMediaSource = nullptr;82MFAPI::CopyImage = nullptr;8384return true;85}8687WindowsCaptureDevice *winCamera;88WindowsCaptureDevice *winMic;8990// TODO: Add more formats, but need some tests.91VideoFormatTransform g_VideoFormats[] =92{93{ MFVideoFormat_RGB32, AV_PIX_FMT_RGBA },94{ MFVideoFormat_RGB24, AV_PIX_FMT_RGB24 },95{ MFVideoFormat_YUY2, AV_PIX_FMT_YUYV422 },96{ MFVideoFormat_NV12, AV_PIX_FMT_NV12 }97};9899AudioFormatTransform g_AudioFormats[] = {100{ MFAudioFormat_PCM, 8, AV_SAMPLE_FMT_U8 },101{ MFAudioFormat_PCM, 16, AV_SAMPLE_FMT_S16 },102{ MFAudioFormat_PCM, 32, AV_SAMPLE_FMT_S32 },103{ MFAudioFormat_Float, 32, AV_SAMPLE_FMT_FLT }104};105106const int g_cVideoFormats = ARRAYSIZE(g_VideoFormats);107const int g_cAudioFormats = ARRAYSIZE(g_AudioFormats);108109MediaParam defaultVideoParam = { { 640, 480, 0, MFVideoFormat_RGB24 } };110MediaParam defaultAudioParam = { { 44100, 2, 16, MFAudioFormat_PCM } };111112HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride);113114ReaderCallback::ReaderCallback(WindowsCaptureDevice *_device) : device(_device) {}115116ReaderCallback::~ReaderCallback() {117#ifdef USE_FFMPEG118if (img_convert_ctx) {119sws_freeContext(img_convert_ctx);120}121if (resample_ctx) {122swr_free(&resample_ctx);123}124#endif125}126127HRESULT ReaderCallback::QueryInterface(REFIID riid, void** ppv)128{129static const QITAB qit[] =130{131QITABENT(ReaderCallback, IMFSourceReaderCallback),132{ 0 },133};134return QISearch(this, qit, riid, ppv);135}136137HRESULT ReaderCallback::OnReadSample(138HRESULT hrStatus,139DWORD dwStreamIndex,140DWORD dwStreamFlags,141LONGLONG llTimestamp,142IMFSample *pSample) {143HRESULT hr = S_OK;144IMFMediaBuffer *pBuffer = nullptr;145std::lock_guard<std::mutex> lock(device->sdMutex);146if (device->isShutDown())147return hr;148149if (FAILED(hrStatus))150hr = hrStatus;151152if (SUCCEEDED(hr)) {153if (pSample) {154hr = pSample->GetBufferByIndex(0, &pBuffer);155}156}157if (SUCCEEDED(hr)) {158switch (device->type) {159case CAPTUREDEVIDE_TYPE::VIDEO: {160BYTE *pbScanline0 = nullptr;161VideoBufferLock *videoBuffer = nullptr;162int imgJpegSize = device->imgJpegSize;163unsigned char* invertedSrcImg = nullptr;164LONG srcPadding = 0;165LONG lStride = 0;166167UINT32 srcW = device->deviceParam.width;168UINT32 srcH = device->deviceParam.height;169UINT32 dstW = device->targetMediaParam.width;170UINT32 dstH = device->targetMediaParam.height;171GUID srcMFVideoFormat = device->deviceParam.videoFormat;172173// pSample can be null, in this case ReadSample still should be called to request next frame.174if (pSample) {175videoBuffer = new VideoBufferLock(pBuffer);176hr = videoBuffer->LockBuffer(device->deviceParam.default_stride, device->deviceParam.height, &pbScanline0, &lStride);177178if (lStride > 0)179srcPadding = lStride - device->deviceParam.default_stride;180else181srcPadding = device->deviceParam.default_stride - lStride;182183#ifdef USE_FFMPEG184if (SUCCEEDED(hr)) {185// Convert image to RGB24186if (lStride > 0) {187imgConvert(188device->imageRGB, dstW, dstH, device->imgRGBLineSizes,189pbScanline0, srcW, srcH, srcMFVideoFormat, srcPadding);190} else {191// If stride < 0, the pointer to the first row of source image is the last row in memory,should invert it in memory.192invertedSrcImg = (unsigned char*)av_malloc(av_image_get_buffer_size(getAVVideoFormatbyMFVideoFormat(srcMFVideoFormat), srcW, srcH, 1));193imgInvert(invertedSrcImg, pbScanline0, srcW, srcH, device->deviceParam.videoFormat, lStride);194// We alloc a inverted image with no padding, set padding to zero.195srcPadding = 0;196imgConvert(197device->imageRGB, dstW, dstH, device->imgRGBLineSizes,198invertedSrcImg, srcW, srcH, srcMFVideoFormat, srcPadding);199av_free(invertedSrcImg);200}201202// Mirror the image in-place if needed.203if (g_Config.bCameraMirrorHorizontal) {204for (int y = 0; y < dstH; y++) {205uint8_t *line = device->imageRGB + y * device->imgRGBLineSizes[0];206for (int x = 0; x < dstW / 2; x++) {207const int invX = dstW - 1 - x;208const uint8_t r = line[x * 3 + 0];209const uint8_t g = line[x * 3 + 1];210const uint8_t b = line[x * 3 + 2];211line[x * 3 + 0] = line[invX * 3 + 0];212line[x * 3 + 1] = line[invX * 3 + 1];213line[x * 3 + 2] = line[invX * 3 + 2];214line[invX * 3 + 0] = r;215line[invX * 3 + 1] = g;216line[invX * 3 + 2] = b;217}218}219}220221// Compress image to jpeg from RGB24.222jpge::compress_image_to_jpeg_file_in_memory(223device->imageJpeg, imgJpegSize,224dstW,225dstH,2263,227device->imageRGB);228}229#endif230Camera::pushCameraImage(imgJpegSize, device->imageJpeg);231}232// Request the next frame.233if (SUCCEEDED(hr)) {234hr = device->m_pReader->ReadSample(235(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,2360,237nullptr,238nullptr,239nullptr,240nullptr241);242}243delete videoBuffer;244break;245}246case CAPTUREDEVIDE_TYPE::Audio: {247BYTE *sampleBuf = nullptr;248DWORD length = 0;249u32 sizeAfterResample = 0;250// pSample can be null, in this case ReadSample still should be called to request next frame.251if (pSample) {252pBuffer->Lock(&sampleBuf, nullptr, &length);253if (device->needResample()) {254sizeAfterResample = doResample(255&device->resampleBuf, device->targetMediaParam.sampleRate, device->targetMediaParam.channels, &device->resampleBufSize,256sampleBuf, device->deviceParam.sampleRate, device->deviceParam.channels, device->deviceParam.audioFormat, length, device->deviceParam.bitsPerSample);257if (device->resampleBuf)258Microphone::addAudioData(device->resampleBuf, sizeAfterResample);259} else {260Microphone::addAudioData(sampleBuf, length);261}262pBuffer->Unlock();263}264// Request the next frame.265if (SUCCEEDED(hr)) {266hr = device->m_pReader->ReadSample(267(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,2680,269nullptr,270nullptr,271nullptr,272nullptr273);274}275break;276}277}278}279280SafeRelease(&pBuffer);281return hr;282}283284AVPixelFormat ReaderCallback::getAVVideoFormatbyMFVideoFormat(const GUID &MFVideoFormat) {285for (int i = 0; i < g_cVideoFormats; i++) {286if (MFVideoFormat == g_VideoFormats[i].MFVideoFormat)287return g_VideoFormats[i].AVVideoFormat;288}289return AV_PIX_FMT_RGB24;290}291292AVSampleFormat ReaderCallback::getAVAudioFormatbyMFAudioFormat(const GUID &MFAudioFormat, const u32 &bitsPerSample) {293for (int i = 0; i < g_cAudioFormats; i++) {294if (MFAudioFormat == g_AudioFormats[i].MFAudioFormat && bitsPerSample == g_AudioFormats[i].bitsPerSample)295return g_AudioFormats[i].AVAudioFormat;296}297return AV_SAMPLE_FMT_S16;298}299300void ReaderCallback::imgConvert(301unsigned char *dst, unsigned int &dstW, unsigned int &dstH, int dstLineSizes[4],302unsigned char *src, const unsigned int &srcW, const unsigned int &srcH, const GUID &srcFormat,303const int &srcPadding) {304#ifdef USE_FFMPEG305int srcLineSizes[4] = { 0, 0, 0, 0 };306unsigned char *pSrc[4];307unsigned char *pDst[4];308309AVPixelFormat srcAVFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);310311312av_image_fill_linesizes(srcLineSizes, srcAVFormat, srcW);313314// Is this correct?315if (srcPadding != 0) {316for (int i = 0; i < 4; i++) {317if (srcLineSizes[i] != 0)318srcLineSizes[i] += srcPadding;319}320}321322av_image_fill_pointers(pSrc, srcAVFormat, srcH, src, srcLineSizes);323av_image_fill_pointers(pDst, AV_PIX_FMT_RGB24, dstH, dst, dstLineSizes);324325if (img_convert_ctx == nullptr) {326img_convert_ctx = sws_getContext(327srcW,328srcH,329srcAVFormat,330dstW,331dstH,332AV_PIX_FMT_RGB24,333SWS_BICUBIC,334nullptr,335nullptr,336nullptr337);338}339340if (img_convert_ctx) {341sws_scale(img_convert_ctx,342(const uint8_t *const *)pSrc,343srcLineSizes,3440,345srcH,346(uint8_t *const *)pDst,347dstLineSizes348);349}350#endif351}352353void ReaderCallback::imgInvert(unsigned char *dst, unsigned char *src, const int &srcW, const int &srcH, const GUID &srcFormat, const int &srcStride) {354#ifdef USE_FFMPEG355AVPixelFormat srcAvFormat = getAVVideoFormatbyMFVideoFormat(srcFormat);356int dstLineSizes[4] = { 0, 0, 0, 0 };357358av_image_fill_linesizes(dstLineSizes, srcAvFormat, srcW);359360if(srcFormat == MFVideoFormat_RGB32)361imgInvertRGBA(dst, dstLineSizes[0], src, srcStride, srcH);362else if(srcFormat == MFVideoFormat_RGB24)363imgInvertRGB(dst, dstLineSizes[0], src, srcStride, srcH);364else if (srcFormat == MFVideoFormat_YUY2)365imgInvertYUY2(dst, dstLineSizes[0], src, srcStride, srcH);366else if (srcFormat == MFVideoFormat_NV12)367imgInvertNV12(dst, dstLineSizes[0], src, srcStride, srcH);;368#endif369}370371void ReaderCallback::imgInvertRGBA(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {372MFAPI::CopyImage(dst, dstStride, src, srcStride, dstStride, h);373}374375void ReaderCallback::imgInvertRGB(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {376for (int y = 0; y < h; y++) {377for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {378dst[dstx] = src[srcx];379}380dst += dstStride;381src += srcStride;382}383}384385void ReaderCallback::imgInvertYUY2(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {386for (int y = 0; y < h; y++) {387for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {388dst[dstx] = src[srcx];389}390dst += dstStride;391src += srcStride;392}393}394395void ReaderCallback::imgInvertNV12(unsigned char *dst, int &dstStride, unsigned char *src, const int &srcStride, const int &h) {396unsigned char *dstY = dst;397unsigned char *dstU = dst + dstStride * h;398unsigned char *srcY = src;399unsigned char *srcV = src + srcStride * h;400401unsigned char *srcY1 = srcY;402unsigned char *srcY2 = srcY1 + srcStride;403unsigned char *dstY1 = dstY;404unsigned char *dstY2 = dstY1 + dstStride;405406bool isodd = h % 2 != 0;407408for (int y = 0; y < (isodd ? h - 1 : h); y += 2) {409for (int srcx = dstStride - 1, dstx = 0; dstx < dstStride; srcx--, dstx++) {410dstY1[dstx] = srcY1[srcx];411dstY2[dstx] = srcY2[srcx];412dstU[dstx] = srcV[srcx];413}414dstY += dstStride * 2;415srcY1 += srcStride * 2;416srcY2 += srcStride *2;417srcV += srcStride;418dstU += dstStride;419}420}421422u32 ReaderCallback::doResample(u8 **dst, u32 &dstSampleRate, u32 &dstChannels, u32 *dstSize, u8 *src, const u32 &srcSampleRate, const u32 &srcChannels, const GUID &srcFormat, const u32& srcSize, const u32& srcBitsPerSample) {423#ifdef USE_FFMPEG424AVSampleFormat srcAVFormat = getAVAudioFormatbyMFAudioFormat(srcFormat, srcBitsPerSample);425int outSamplesCount = 0;426if (resample_ctx == nullptr) {427resample_ctx = swr_alloc_set_opts(nullptr,428av_get_default_channel_layout(dstChannels),429AV_SAMPLE_FMT_S16,430dstSampleRate,431av_get_default_channel_layout(srcChannels),432srcAVFormat,433srcSampleRate,4340,435nullptr);436if (resample_ctx == nullptr || swr_init(resample_ctx) < 0) {437swr_free(&resample_ctx);438return 0;439}440}441int srcSamplesCount = srcSize / srcChannels / av_get_bytes_per_sample(srcAVFormat); // per channel.442int outCount = srcSamplesCount * dstSampleRate / srcSampleRate + 256;443unsigned int outSize = av_samples_get_buffer_size(nullptr, dstChannels, outCount, AV_SAMPLE_FMT_S16, 0);444445if (!*dst) {446*dst = (u8 *)av_malloc(outSize);447*dstSize = outSize;448}449if (!*dst)450return 0;451452if(*dstSize < outSize)453av_fast_malloc(dst, dstSize, outSize);454455outSamplesCount = swr_convert(resample_ctx, dst, outCount, (const uint8_t **)&src, srcSamplesCount);456if (outSamplesCount < 0)457return 0;458return av_samples_get_buffer_size(nullptr, dstChannels, outSamplesCount, AV_SAMPLE_FMT_S16, 0);459#else460return 0;461#endif462}463464WindowsCaptureDevice::WindowsCaptureDevice(CAPTUREDEVIDE_TYPE _type) :465type(_type),466error(CAPTUREDEVIDE_ERROR_NO_ERROR),467state(CAPTUREDEVIDE_STATE::UNINITIALIZED) {468param = { 0 };469deviceParam = { { 0 } };470471switch (type) {472case CAPTUREDEVIDE_TYPE::VIDEO:473targetMediaParam = defaultVideoParam;474break;475case CAPTUREDEVIDE_TYPE::Audio:476targetMediaParam = defaultAudioParam;477break;478}479480std::thread t(&WindowsCaptureDevice::messageHandler, this);481t.detach();482}483484WindowsCaptureDevice::~WindowsCaptureDevice() {485#ifdef USE_FFMPEG486switch (type) {487case CAPTUREDEVIDE_TYPE::VIDEO:488av_freep(&imageRGB);489av_freep(&imageJpeg);490break;491case CAPTUREDEVIDE_TYPE::Audio:492av_freep(&resampleBuf);493break;494}495#endif496}497498void WindowsCaptureDevice::CheckDevices() {499isDeviceChanged = true;500}501502bool WindowsCaptureDevice::init() {503HRESULT hr = S_OK;504505if (!RegisterCMPTMFApis()) {506setError(CAPTUREDEVIDE_ERROR_INIT_FAILED, "Cannot register devices");507return false;508}509std::unique_lock<std::mutex> lock(paramMutex);510hr = enumDevices();511lock.unlock();512513if (FAILED(hr)) {514setError(CAPTUREDEVIDE_ERROR_INIT_FAILED, "Cannot enumerate devices");515return false;516}517518updateState(CAPTUREDEVIDE_STATE::STOPPED);519return true;520}521522bool WindowsCaptureDevice::start(void *startParam) {523HRESULT hr = S_OK;524IMFAttributes *pAttributes = nullptr;525IMFMediaType *pType = nullptr;526UINT32 selection = 0;527UINT32 count = 0;528529// Release old sources first(if any).530SafeRelease(&m_pSource);531SafeRelease(&m_pReader);532if (m_pCallback) {533delete m_pCallback;534m_pCallback = nullptr;535}536// Need to re-enumerate the list,because old sources were released.537std::vector<std::string> deviceList = getDeviceList(true);538539if (deviceList.size() < 1) {540setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Has no device");541return false;542}543544m_pCallback = new ReaderCallback(this);545546std::string selectedDeviceName = type == CAPTUREDEVIDE_TYPE::VIDEO ? g_Config.sCameraDevice : g_Config.sMicDevice;547548switch (state) {549case CAPTUREDEVIDE_STATE::STOPPED:550for (auto &name : deviceList) {551if (name == selectedDeviceName) {552selection = count;553break;554}555++count;556}557setSelction(selection);558hr = param.ppDevices[param.selection]->ActivateObject(559__uuidof(IMFMediaSource),560(void**)&m_pSource);561562if (SUCCEEDED(hr))563hr = MFCreateAttributes(&pAttributes, 2);564565// Use async mode566if (SUCCEEDED(hr))567hr = pAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, m_pCallback);568569if (SUCCEEDED(hr))570hr = pAttributes->SetUINT32(MF_READWRITE_DISABLE_CONVERTERS, TRUE);571572if (SUCCEEDED(hr)) {573hr = CreateSourceReaderFromMediaSource(574m_pSource,575pAttributes,576&m_pReader577);578}579580if (!m_pReader)581hr = E_FAIL;582583if (SUCCEEDED(hr)) {584switch (type) {585case CAPTUREDEVIDE_TYPE::VIDEO: {586if (startParam) {587std::vector<int> *resolution = static_cast<std::vector<int>*>(startParam);588targetMediaParam.width = resolution->at(0);589targetMediaParam.height = resolution->at(1);590delete resolution;591}592#ifdef USE_FFMPEG593594av_freep(&imageRGB);595av_freep(&imageJpeg);596imageRGB = (unsigned char*)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_RGB24, targetMediaParam.width, targetMediaParam.height, 1));597imgJpegSize = av_image_get_buffer_size(AV_PIX_FMT_YUVJ411P, targetMediaParam.width, targetMediaParam.height, 1);598imageJpeg = (unsigned char*)av_malloc(imgJpegSize);599av_image_fill_linesizes(imgRGBLineSizes, AV_PIX_FMT_RGB24, targetMediaParam.width);600#endif601602for (DWORD i = 0; ; i++) {603hr = m_pReader->GetNativeMediaType(604(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,605i,606&pType607);608609if (FAILED(hr)) { break; }610611hr = setDeviceParam(pType);612613if (SUCCEEDED(hr))614break;615}616/*617hr = m_pReader->GetNativeMediaType(618(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,619(DWORD)0xFFFFFFFF,//MF_SOURCE_READER_CURRENT_TYPE_INDEX620&pType621);622if (SUCCEEDED(hr))623hr = setDeviceParam(pType);*/ // Don't support on Win7624625// Request the first frame, in async mode, OnReadSample will be called when ReadSample completed.626if (SUCCEEDED(hr)) {627hr = m_pReader->ReadSample(628(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,6290,630nullptr,631nullptr,632nullptr,633nullptr634);635}636break;637}638639case CAPTUREDEVIDE_TYPE::Audio: {640if (startParam) {641std::vector<u32> *micParam = static_cast<std::vector<u32>*>(startParam);642targetMediaParam.sampleRate = micParam->at(0);643targetMediaParam.channels = micParam->at(1);644delete micParam;645}646647for (DWORD i = 0; ; i++) {648hr = m_pReader->GetNativeMediaType(649(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,650i,651&pType652);653654if (FAILED(hr)) { break; }655656hr = setDeviceParam(pType);657658if (SUCCEEDED(hr))659break;660}661662if (SUCCEEDED(hr)) {663hr = m_pReader->ReadSample(664(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,6650,666nullptr,667nullptr,668nullptr,669nullptr670);671}672break;673}674}675}676677if (FAILED(hr)) {678setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Cannot start");679if(m_pSource)680m_pSource->Shutdown();681SafeRelease(&m_pSource);682SafeRelease(&pAttributes);683SafeRelease(&pType);684SafeRelease(&m_pReader);685return false;686}687688SafeRelease(&pAttributes);689SafeRelease(&pType);690updateState(CAPTUREDEVIDE_STATE::STARTED);691break;692case CAPTUREDEVIDE_STATE::LOST:693setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device has lost");694return false;695case CAPTUREDEVIDE_STATE::STARTED:696setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device has started");697return false;698case CAPTUREDEVIDE_STATE::UNINITIALIZED:699setError(CAPTUREDEVIDE_ERROR_START_FAILED, "Device doesn't initialize");700return false;701default:702break;703}704return true;705}706707bool WindowsCaptureDevice::stop() {708if (state == CAPTUREDEVIDE_STATE::STOPPED)709return true;710if (m_pSource)711m_pSource->Stop();712713updateState(CAPTUREDEVIDE_STATE::STOPPED);714715return true;716};717718std::vector<std::string> WindowsCaptureDevice::getDeviceList(bool forceEnum, int *pActuallCount) {719HRESULT hr = S_OK;720UINT32 count = 0;721LPWSTR pwstrName = nullptr;722char *cstrName = nullptr;723std::string strName;724DWORD dwMinSize = 0;725std::vector<std::string> deviceList;726727if (isDeviceChanged || forceEnum) {728std::unique_lock<std::mutex> lock(paramMutex);729for (DWORD i = 0; i < param.count; i++) {730SafeRelease(¶m.ppDevices[i]);731}732CoTaskMemFree(param.ppDevices); // Null pointer is okay.733734hr = enumDevices();735736lock.unlock();737738if (SUCCEEDED(hr))739isDeviceChanged = false;740else741return deviceList;742}743744for (; count < param.count; count++) {745hr = param.ppDevices[count]->GetAllocatedString(746MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME,747&pwstrName,748nullptr749);750751if (SUCCEEDED(hr)) {752// Get the size needed first753dwMinSize = WideCharToMultiByte(CP_UTF8, NULL, pwstrName, -1, nullptr, 0, nullptr, FALSE);754if (dwMinSize == 0)755hr = E_FAIL;756}757if (SUCCEEDED(hr)) {758cstrName = new char[dwMinSize];759WideCharToMultiByte(CP_UTF8, NULL, pwstrName, -1, cstrName, dwMinSize, NULL, FALSE);760strName = cstrName;761delete[] cstrName;762763deviceList.push_back(strName);764}765766CoTaskMemFree(pwstrName);767768if (FAILED(hr)) {769setError(CAPTUREDEVIDE_ERROR_GETNAMES_FAILED, "Error occurred,gotten " + std::to_string((int)count) + " device names");770if(pActuallCount)771*pActuallCount = count;772return deviceList;773}774}775if (pActuallCount)776*pActuallCount = count + 1;777return deviceList;778}779780HRESULT WindowsCaptureDevice::setDeviceParam(IMFMediaType *pType) {781HRESULT hr = S_OK;782GUID subtype = { 0 };783bool getFormat = false;784785switch (type) {786case CAPTUREDEVIDE_TYPE::VIDEO:787hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);788if (FAILED(hr))789break;790791for (int i = 0; i < g_cVideoFormats; i++) {792if (subtype == g_VideoFormats[i].MFVideoFormat) {793deviceParam.videoFormat = subtype;794getFormat = true;795break;796}797}798799if (!getFormat) {800for (int i = 0; i < g_cVideoFormats; i++) {801hr = pType->SetGUID(MF_MT_SUBTYPE, g_VideoFormats[i].MFVideoFormat);802if (FAILED(hr))803continue;804805hr = m_pReader->SetCurrentMediaType(806(DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM,807NULL,808pType809);810811if (SUCCEEDED(hr)) {812deviceParam.videoFormat = g_VideoFormats[i].MFVideoFormat;813getFormat = true;814break;815}816}817}818if (SUCCEEDED(hr))819hr = MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &deviceParam.width, &deviceParam.height);820821if (SUCCEEDED(hr))822hr = GetDefaultStride(pType, &deviceParam.default_stride);823824break;825case CAPTUREDEVIDE_TYPE::Audio:826hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);827if (FAILED(hr))828break;829830for (int i = 0; i < g_cVideoFormats; i++) {831if (subtype == g_AudioFormats[i].MFAudioFormat) {832deviceParam.audioFormat = subtype;833getFormat = true;834break;835}836}837838if (!getFormat) {839for (int i = 0; i < g_cAudioFormats; i++) {840hr = pType->SetGUID(MF_MT_SUBTYPE, g_AudioFormats[i].MFAudioFormat);841if (FAILED(hr))842continue;843844hr = m_pReader->SetCurrentMediaType(845(DWORD)MF_SOURCE_READER_FIRST_AUDIO_STREAM,846NULL,847pType848);849850if (SUCCEEDED(hr)) {851deviceParam.audioFormat = g_AudioFormats[i].MFAudioFormat;852getFormat = true;853break;854}855}856}857if (SUCCEEDED(hr))858hr = pType->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, &deviceParam.sampleRate);859860if (SUCCEEDED(hr))861hr = pType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &deviceParam.channels);862863if (SUCCEEDED(hr))864hr = pType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, (UINT32 *)&deviceParam.bitsPerSample);865866break;867}868869return hr;870}871872void WindowsCaptureDevice::sendMessage(CAPTUREDEVIDE_MESSAGE message) {873// Must be unique lock874std::unique_lock<std::mutex> lock(mutex);875messageQueue.push(message);876cond.notify_one();877}878879CAPTUREDEVIDE_MESSAGE WindowsCaptureDevice::getMessage() {880// Must be unique lock881std::unique_lock<std::mutex> lock(mutex);882CAPTUREDEVIDE_MESSAGE message;883cond.wait(lock, [this]() { return !messageQueue.empty(); });884message = messageQueue.front();885messageQueue.pop();886887return message;888}889890void WindowsCaptureDevice::updateState(const CAPTUREDEVIDE_STATE &newState) {891state = newState;892if (isShutDown()) {893std::unique_lock<std::mutex> guard(stateMutex_);894stateCond_.notify_all();895}896}897898void WindowsCaptureDevice::waitShutDown() {899sendMessage({ CAPTUREDEVIDE_COMMAND::SHUTDOWN, nullptr });900901std::unique_lock<std::mutex> guard(stateMutex_);902while (!isShutDown()) {903stateCond_.wait(guard);904}905}906907void WindowsCaptureDevice::messageHandler() {908CoInitializeEx(NULL, COINIT_MULTITHREADED);909MFStartup(MF_VERSION);910CAPTUREDEVIDE_MESSAGE message;911912if (type == CAPTUREDEVIDE_TYPE::VIDEO) {913SetCurrentThreadName("Camera");914} else if (type == CAPTUREDEVIDE_TYPE::Audio) {915SetCurrentThreadName("Microphone");916}917918while ((message = getMessage()).command != CAPTUREDEVIDE_COMMAND::SHUTDOWN) {919switch (message.command) {920case CAPTUREDEVIDE_COMMAND::INITIALIZE:921init();922break;923case CAPTUREDEVIDE_COMMAND::START:924start(message.opacity);925break;926case CAPTUREDEVIDE_COMMAND::STOP:927stop();928break;929case CAPTUREDEVIDE_COMMAND::UPDATE_STATE:930updateState((*(CAPTUREDEVIDE_STATE *)message.opacity));931break;932}933}934935if (state != CAPTUREDEVIDE_STATE::STOPPED)936stop();937938std::lock_guard<std::mutex> lock(sdMutex);939SafeRelease(&m_pSource);940SafeRelease(&m_pReader);941delete m_pCallback;942unRegisterCMPTMFApis();943944std::unique_lock<std::mutex> lock2(paramMutex);945for (DWORD i = 0; i < param.count; i++) {946SafeRelease(¶m.ppDevices[i]);947}948CoTaskMemFree(param.ppDevices); // Null pointer is okay.949lock2.unlock();950951MFShutdown();952CoUninitialize();953954updateState(CAPTUREDEVIDE_STATE::SHUTDOWN);955}956957HRESULT WindowsCaptureDevice::enumDevices() {958HRESULT hr = S_OK;959IMFAttributes *pAttributes = nullptr;960961hr = MFCreateAttributes(&pAttributes, 1);962if (SUCCEEDED(hr)) {963switch (type) {964case CAPTUREDEVIDE_TYPE::VIDEO:965hr = pAttributes->SetGUID(966MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,967MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID968);969970break;971case CAPTUREDEVIDE_TYPE::Audio:972hr = pAttributes->SetGUID(973MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,974MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID975);976977break;978default:979setError(CAPTUREDEVIDE_ERROR_UNKNOWN_TYPE, "Unknown device type");980return E_FAIL;981}982}983if (SUCCEEDED(hr)) {984hr = EnumDeviceSources(pAttributes, ¶m.ppDevices, ¶m.count);985}986987SafeRelease(&pAttributes);988return hr;989}990991bool WindowsCaptureDevice::needResample() {992return deviceParam.sampleRate != targetMediaParam.sampleRate ||993deviceParam.channels != targetMediaParam.channels ||994deviceParam.audioFormat != targetMediaParam.audioFormat ||995deviceParam.bitsPerSample != targetMediaParam.bitsPerSample;996}997998//-----------------------------------------------------------------------------999// GetDefaultStride1000//1001// Gets the default stride for a video frame, assuming no extra padding bytes.1002//1003//-----------------------------------------------------------------------------10041005HRESULT GetDefaultStride(IMFMediaType *pType, LONG *plStride)1006{1007LONG lStride = 0;10081009// Try to get the default stride from the media type.1010HRESULT hr = pType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&lStride);1011if (FAILED(hr))1012{1013// Attribute not set. Try to calculate the default stride.1014GUID subtype = GUID_NULL;10151016UINT32 width = 0;1017UINT32 height = 0;10181019// Get the subtype and the image size.1020hr = pType->GetGUID(MF_MT_SUBTYPE, &subtype);1021if (SUCCEEDED(hr))1022{1023hr = MFGetAttributeSize(pType, MF_MT_FRAME_SIZE, &width, &height);1024}1025if (SUCCEEDED(hr))1026{1027hr = GetStrideForBitmapInfoHeader(subtype.Data1, width, &lStride);1028}10291030// Set the attribute for later reference.1031if (SUCCEEDED(hr))1032{1033(void)pType->SetUINT32(MF_MT_DEFAULT_STRIDE, UINT32(lStride));1034}1035}10361037if (SUCCEEDED(hr))1038{1039*plStride = lStride;1040}1041return hr;1042}104310441045