diff --git a/runtime/core/bin/CMakeLists.txt b/runtime/core/bin/CMakeLists.txt new file mode 100644 index 0000000..02eb0eb --- /dev/null +++ b/runtime/core/bin/CMakeLists.txt @@ -0,0 +1,5 @@ +add_executable(kws_main kws_main.cc) +target_link_libraries(kws_main PUBLIC onnxruntime frontend kws) + +add_executable(stream_kws_main stream_kws_main.cc) +target_link_libraries(stream_kws_main PUBLIC onnxruntime frontend kws portaudio_static) diff --git a/runtime/core/bin/stream_kws_main.cc b/runtime/core/bin/stream_kws_main.cc new file mode 100644 index 0000000..e7dec44 --- /dev/null +++ b/runtime/core/bin/stream_kws_main.cc @@ -0,0 +1,106 @@ +// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include "portaudio.h" // NOLINT + +#include "frontend/feature_pipeline.h" +#include "frontend/wav.h" +#include "kws/keyword_spotting.h" +#include "utils/log.h" + +int g_exiting = 0; +std::shared_ptr g_feature_pipeline; + +void SigRoutine(int dunno) { + if (dunno == SIGINT) { + g_exiting = 1; + } +} + +static int RecordCallback(const void* input, void* output, + unsigned long frames_count, // NOLINT + const PaStreamCallbackTimeInfo* time_info, + PaStreamCallbackFlags status_flags, void* user_data) { + const auto* pcm_data = static_cast(input); + std::vector v(pcm_data, pcm_data + frames_count); + g_feature_pipeline->AcceptWaveform(v); + + if (g_exiting) { + LOG(INFO) << "Exiting loop."; + g_feature_pipeline->set_input_finished(); + return paComplete; + } else { + return paContinue; + } +} + +int main(int argc, char* argv[]) { + if (argc != 4) { + LOG(FATAL) << "Usage: kws_main fbank_dim batch_size kws_model_path"; + } + const int num_bins = std::stoi(argv[1]); // Fbank feature dim + const int batch_size = std::stoi(argv[2]); + const std::string model_path = argv[3]; + + wenet::FeaturePipelineConfig feature_config(num_bins, 16000); + g_feature_pipeline = std::make_shared(feature_config); + wekws::KeywordSpotting spotter(model_path); + + signal(SIGINT, SigRoutine); + PaError err = Pa_Initialize(); + PaStreamParameters params; + std::cout << err << " " << Pa_GetDeviceCount() << std::endl; + params.device = Pa_GetDefaultInputDevice(); + if (params.device == paNoDevice) { + LOG(FATAL) << "Error: No default input device."; + } + params.channelCount = 1; + params.sampleFormat = paInt16; + params.suggestedLatency = + Pa_GetDeviceInfo(params.device)->defaultLowInputLatency; + params.hostApiSpecificStreamInfo = NULL; + PaStream* stream; + // Callback and spot pcm date each `interval` ms. + int interval = 500; + int frames_per_buffer = 16000 / 1000 * interval; + Pa_OpenStream(&stream, ¶ms, NULL, 16000, frames_per_buffer, paClipOff, + RecordCallback, NULL); + Pa_StartStream(stream); + LOG(INFO) << "=== Now recording!! Please speak into the microphone. ==="; + + std::cout << std::setiosflags(std::ios::fixed) << std::setprecision(2); + while (Pa_IsStreamActive(stream)) { + Pa_Sleep(interval); + std::vector> feats; + g_feature_pipeline->Read(batch_size, &feats); + std::vector> prob; + spotter.Forward(feats, &prob); + for (int t = 0; t < prob.size(); t++) { + std::cout << "keywords prob:"; + for (int i = 0; i < prob[t].size(); i++) { + std::cout << " kw[" << i << "] " << prob[t][i]; + } + std::cout << std::endl; + } + } + Pa_CloseStream(stream); + Pa_Terminate(); + + return 0; +} diff --git a/runtime/core/cmake/onnxruntime.cmake b/runtime/core/cmake/onnxruntime.cmake new file mode 100644 index 0000000..d391713 --- /dev/null +++ b/runtime/core/cmake/onnxruntime.cmake @@ -0,0 +1,16 @@ +set(ONNX_VERSION "1.12.0") +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-aarch64-${ONNX_VERSION}.tgz") + set(URL_HASH "SHA256=5820d9f343df73c63b6b2b174a1ff62575032e171c9564bcf92060f46827d0ac") +else() + set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz") + set(URL_HASH "SHA256=5d503ce8540358b59be26c675e42081be14a3e833a5301926f555451046929c5") +endif() + +FetchContent_Declare(onnxruntime + URL ${ONNX_URL} + URL_HASH ${URL_HASH} +) +FetchContent_MakeAvailable(onnxruntime) +include_directories(${onnxruntime_SOURCE_DIR}/include) +link_directories(${onnxruntime_SOURCE_DIR}/lib) diff --git a/runtime/core/cmake/portaudio.cmake b/runtime/core/cmake/portaudio.cmake new file mode 100644 index 0000000..ae5bc37 --- /dev/null +++ b/runtime/core/cmake/portaudio.cmake @@ -0,0 +1,6 @@ +FetchContent_Declare(portaudio + URL https://github.com/PortAudio/portaudio/archive/refs/tags/v19.7.0.tar.gz + URL_HASH SHA256=5af29ba58bbdbb7bbcefaaecc77ec8fc413f0db6f4c4e286c40c3e1b83174fa0 +) +FetchContent_MakeAvailable(portaudio) +include_directories(${portaudio_SOURCE_DIR}/include) diff --git a/runtime/core/frontend/CMakeLists.txt b/runtime/core/frontend/CMakeLists.txt new file mode 100644 index 0000000..006e9eb --- /dev/null +++ b/runtime/core/frontend/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(frontend STATIC + feature_pipeline.cc + fft.cc +) diff --git a/runtime/core/kws/CMakeLists.txt b/runtime/core/kws/CMakeLists.txt new file mode 100644 index 0000000..bc92236 --- /dev/null +++ b/runtime/core/kws/CMakeLists.txt @@ -0,0 +1 @@ +add_library(kws STATIC keyword_spotting.cc) diff --git a/runtime/core/toolchains/aarch64-linux-gnu.toolchain.cmake b/runtime/core/toolchains/aarch64-linux-gnu.toolchain.cmake new file mode 100644 index 0000000..948164d --- /dev/null +++ b/runtime/core/toolchains/aarch64-linux-gnu.toolchain.cmake @@ -0,0 +1,5 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) + +set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) diff --git a/runtime/onnxruntime/CMakeLists.txt b/runtime/onnxruntime/CMakeLists.txt index 4964dcd..0fcd6dc 100644 --- a/runtime/onnxruntime/CMakeLists.txt +++ b/runtime/onnxruntime/CMakeLists.txt @@ -5,27 +5,16 @@ project(wekws VERSION 0.1) set(CMAKE_VERBOSE_MAKEFILE on) include(FetchContent) -include(ExternalProject) set(FETCHCONTENT_QUIET OFF) get_filename_component(fc_base "fc_base" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") set(FETCHCONTENT_BASE_DIR ${fc_base}) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -g -pthread") include_directories(${CMAKE_CURRENT_SOURCE_DIR}) -FetchContent_Declare(onnxruntime - URL https://github.com/microsoft/onnxruntime/releases/download/v1.11.1/onnxruntime-linux-x64-1.11.1.tgz - URL_HASH SHA256=ddc03b5ae325c675ff76a6f18786ce7d310be6eb6f320087f7a0e9228115f24d -) -FetchContent_MakeAvailable(onnxruntime) -include_directories(${onnxruntime_SOURCE_DIR}/include) -link_directories(${onnxruntime_SOURCE_DIR}/lib) - - -add_executable(kws_main - bin/kws_main.cc - kws/keyword_spotting.cc - frontend/feature_pipeline.cc - frontend/fft.cc -) -target_link_libraries(kws_main PUBLIC onnxruntime) +include(portaudio) +include(onnxruntime) +add_subdirectory(frontend) +add_subdirectory(kws) +add_subdirectory(bin) diff --git a/runtime/onnxruntime/cmake b/runtime/onnxruntime/cmake new file mode 120000 index 0000000..17afee8 --- /dev/null +++ b/runtime/onnxruntime/cmake @@ -0,0 +1 @@ +../core/cmake \ No newline at end of file diff --git a/runtime/raspberrypi/CMakeLists.txt b/runtime/raspberrypi/CMakeLists.txt new file mode 120000 index 0000000..19f9642 --- /dev/null +++ b/runtime/raspberrypi/CMakeLists.txt @@ -0,0 +1 @@ +../onnxruntime/CMakeLists.txt \ No newline at end of file diff --git a/runtime/raspberrypi/README.md b/runtime/raspberrypi/README.md new file mode 100644 index 0000000..64e5890 --- /dev/null +++ b/runtime/raspberrypi/README.md @@ -0,0 +1,44 @@ +# WeNet & Raspberry PI + +There are two ways to build the runtime binaries for Raspberry PI. + +1. Refer `runtime/onnxruntime/README.md` to build it in Raspberry PI. +2. Cross compile and `scp` the binaries and libraries to Raspberry PI. + +## Cross Compile + +* Step 1. Install cross compile tools in the PC. + +``` sh +sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu +``` + +Or download, and install the binaries from: https://releases.linaro.org/components/toolchain/binaries/latest-7 + + +* Step 2. Export your experiment model to ONNX by https://github.com/wenet-e2e/wekws/blob/main/wekws/bin/export_onnx.py + +``` sh +exp=exp # Change it to your experiment dir +python -m wekws.bin.export_onnx \ + --config $exp/train.yaml \ + --checkpoint $exp/final.pt \ + --output_dir final.onnx +``` + +* Step 3. Build. The build requires cmake 3.14 or above. and Send the binary and libraries to Raspberry PI. + +``` sh +cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=toolchains/aarch64-linux-gnu.toolchain.cmake +cmake --build build +scp -r build/bin pi@xxx.xxx.xxx:/path/to/wekws +scp fc_base/onnxruntime-src/lib/libonnxruntime.so* pi@xxx.xxx.xxx:/path/to/wekws +``` + +* Step 4. Run. The log will be shown in Raspberry PI's console. + +``` sh +cd /path/to/wekws +export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH +./build/bin/stream_kws_main 40 80 final.onnx +``` diff --git a/runtime/raspberrypi/bin b/runtime/raspberrypi/bin new file mode 120000 index 0000000..938df72 --- /dev/null +++ b/runtime/raspberrypi/bin @@ -0,0 +1 @@ +../core/bin \ No newline at end of file diff --git a/runtime/raspberrypi/cmake b/runtime/raspberrypi/cmake new file mode 120000 index 0000000..17afee8 --- /dev/null +++ b/runtime/raspberrypi/cmake @@ -0,0 +1 @@ +../core/cmake \ No newline at end of file diff --git a/runtime/raspberrypi/frontend b/runtime/raspberrypi/frontend new file mode 120000 index 0000000..80fb04c --- /dev/null +++ b/runtime/raspberrypi/frontend @@ -0,0 +1 @@ +../core/frontend/ \ No newline at end of file diff --git a/runtime/raspberrypi/kws b/runtime/raspberrypi/kws new file mode 120000 index 0000000..c0da85b --- /dev/null +++ b/runtime/raspberrypi/kws @@ -0,0 +1 @@ +../core/kws/ \ No newline at end of file diff --git a/runtime/raspberrypi/toolchains b/runtime/raspberrypi/toolchains new file mode 120000 index 0000000..5e96b19 --- /dev/null +++ b/runtime/raspberrypi/toolchains @@ -0,0 +1 @@ +../core/toolchains/ \ No newline at end of file diff --git a/runtime/raspberrypi/utils b/runtime/raspberrypi/utils new file mode 120000 index 0000000..0353c24 --- /dev/null +++ b/runtime/raspberrypi/utils @@ -0,0 +1 @@ +../core/utils/ \ No newline at end of file