[runtime] support raspberry pi (#99)

* [runtime] support raspberry pi

* [runtime] add readme for raspberry pi

runtime/core/bin/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_executable(kws_main kws_main.cc)
+target_link_libraries(kws_main PUBLIC onnxruntime frontend kws)
+
+add_executable(stream_kws_main stream_kws_main.cc)
+target_link_libraries(stream_kws_main PUBLIC onnxruntime frontend kws portaudio_static)

runtime/core/bin/stream_kws_main.cc

@@ -0,0 +1,106 @@
+// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn)
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <signal.h>
+#include <iomanip>
+#include <iostream>
+#include <string>
+
+#include "portaudio.h"  // NOLINT
+
+#include "frontend/feature_pipeline.h"
+#include "frontend/wav.h"
+#include "kws/keyword_spotting.h"
+#include "utils/log.h"
+
+int g_exiting = 0;
+std::shared_ptr<wenet::FeaturePipeline> g_feature_pipeline;
+
+void SigRoutine(int dunno) {
+  if (dunno == SIGINT) {
+    g_exiting = 1;
+  }
+}
+
+static int RecordCallback(const void* input, void* output,
+                          unsigned long frames_count,  // NOLINT
+                          const PaStreamCallbackTimeInfo* time_info,
+                          PaStreamCallbackFlags status_flags, void* user_data) {
+  const auto* pcm_data = static_cast<const int16_t*>(input);
+  std::vector<int16_t> v(pcm_data, pcm_data + frames_count);
+  g_feature_pipeline->AcceptWaveform(v);
+
+  if (g_exiting) {
+    LOG(INFO) << "Exiting loop.";
+    g_feature_pipeline->set_input_finished();
+    return paComplete;
+  } else {
+    return paContinue;
+  }
+}
+
+int main(int argc, char* argv[]) {
+  if (argc != 4) {
+    LOG(FATAL) << "Usage: kws_main fbank_dim batch_size kws_model_path";
+  }
+  const int num_bins = std::stoi(argv[1]);  // Fbank feature dim
+  const int batch_size = std::stoi(argv[2]);
+  const std::string model_path = argv[3];
+
+  wenet::FeaturePipelineConfig feature_config(num_bins, 16000);
+  g_feature_pipeline = std::make_shared<wenet::FeaturePipeline>(feature_config);
+  wekws::KeywordSpotting spotter(model_path);
+
+  signal(SIGINT, SigRoutine);
+  PaError err = Pa_Initialize();
+  PaStreamParameters params;
+  std::cout << err << " " << Pa_GetDeviceCount() << std::endl;
+  params.device = Pa_GetDefaultInputDevice();
+  if (params.device == paNoDevice) {
+    LOG(FATAL) << "Error: No default input device.";
+  }
+  params.channelCount = 1;
+  params.sampleFormat = paInt16;
+  params.suggestedLatency =
+      Pa_GetDeviceInfo(params.device)->defaultLowInputLatency;
+  params.hostApiSpecificStreamInfo = NULL;
+  PaStream* stream;
+  // Callback and spot pcm date each `interval` ms.
+  int interval = 500;
+  int frames_per_buffer = 16000 / 1000 * interval;
+  Pa_OpenStream(&stream, &params, NULL, 16000, frames_per_buffer, paClipOff,
+                RecordCallback, NULL);
+  Pa_StartStream(stream);
+  LOG(INFO) << "=== Now recording!! Please speak into the microphone. ===";
+
+  std::cout << std::setiosflags(std::ios::fixed) << std::setprecision(2);
+  while (Pa_IsStreamActive(stream)) {
+    Pa_Sleep(interval);
+    std::vector<std::vector<float>> feats;
+    g_feature_pipeline->Read(batch_size, &feats);
+    std::vector<std::vector<float>> prob;
+    spotter.Forward(feats, &prob);
+    for (int t = 0; t < prob.size(); t++) {
+      std::cout << "keywords prob:";
+      for (int i = 0; i < prob[t].size(); i++) {
+        std::cout << " kw[" << i << "] " << prob[t][i];
+      }
+      std::cout << std::endl;
+    }
+  }
+  Pa_CloseStream(stream);
+  Pa_Terminate();
+
+  return 0;
+}

runtime/core/cmake/onnxruntime.cmake

@@ -0,0 +1,16 @@
+set(ONNX_VERSION "1.12.0")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+  set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-aarch64-${ONNX_VERSION}.tgz")
+  set(URL_HASH "SHA256=5820d9f343df73c63b6b2b174a1ff62575032e171c9564bcf92060f46827d0ac")
+else()
+  set(ONNX_URL "https://github.com/microsoft/onnxruntime/releases/download/v${ONNX_VERSION}/onnxruntime-linux-x64-${ONNX_VERSION}.tgz")
+  set(URL_HASH "SHA256=5d503ce8540358b59be26c675e42081be14a3e833a5301926f555451046929c5")
+endif()
+
+FetchContent_Declare(onnxruntime
+  URL ${ONNX_URL}
+  URL_HASH ${URL_HASH}
+)
+FetchContent_MakeAvailable(onnxruntime)
+include_directories(${onnxruntime_SOURCE_DIR}/include)
+link_directories(${onnxruntime_SOURCE_DIR}/lib)

runtime/core/cmake/portaudio.cmake

@@ -0,0 +1,6 @@
+FetchContent_Declare(portaudio
+  URL      https://github.com/PortAudio/portaudio/archive/refs/tags/v19.7.0.tar.gz
+  URL_HASH SHA256=5af29ba58bbdbb7bbcefaaecc77ec8fc413f0db6f4c4e286c40c3e1b83174fa0
+)
+FetchContent_MakeAvailable(portaudio)
+include_directories(${portaudio_SOURCE_DIR}/include)

runtime/core/frontend/CMakeLists.txt

@@ -0,0 +1,4 @@
+add_library(frontend STATIC
+  feature_pipeline.cc
+  fft.cc
+)

runtime/core/kws/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(kws STATIC keyword_spotting.cc)

runtime/core/toolchains/aarch64-linux-gnu.toolchain.cmake

@@ -0,0 +1,5 @@
+set(CMAKE_SYSTEM_NAME Linux)
+set(CMAKE_SYSTEM_PROCESSOR aarch64)
+
+set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc)
+set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++)

runtime/onnxruntime/CMakeLists.txt

@@ -5,27 +5,16 @@ project(wekws VERSION 0.1)
 set(CMAKE_VERBOSE_MAKEFILE on)
 
 include(FetchContent)
-include(ExternalProject)
 set(FETCHCONTENT_QUIET OFF)
 get_filename_component(fc_base "fc_base" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(FETCHCONTENT_BASE_DIR ${fc_base})
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14 -g -pthread")
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
-FetchContent_Declare(onnxruntime
-  URL      https://github.com/microsoft/onnxruntime/releases/download/v1.11.1/onnxruntime-linux-x64-1.11.1.tgz
-  URL_HASH SHA256=ddc03b5ae325c675ff76a6f18786ce7d310be6eb6f320087f7a0e9228115f24d
-)
-FetchContent_MakeAvailable(onnxruntime)
-include_directories(${onnxruntime_SOURCE_DIR}/include)
-link_directories(${onnxruntime_SOURCE_DIR}/lib)
-
-
-add_executable(kws_main
-  bin/kws_main.cc
-  kws/keyword_spotting.cc
-  frontend/feature_pipeline.cc
-  frontend/fft.cc
-)
-target_link_libraries(kws_main PUBLIC onnxruntime)
+include(portaudio)
+include(onnxruntime)
+add_subdirectory(frontend)
+add_subdirectory(kws)
+add_subdirectory(bin)

runtime/onnxruntime/cmake

@@ -0,0 +1 @@
+../core/cmake

runtime/raspberrypi/CMakeLists.txt

@@ -0,0 +1 @@
+../onnxruntime/CMakeLists.txt

runtime/raspberrypi/README.md

@@ -0,0 +1,44 @@
+# WeNet & Raspberry PI
+
+There are two ways to build the runtime binaries for Raspberry PI.
+
+1. Refer `runtime/onnxruntime/README.md` to build it in Raspberry PI.
+2. Cross compile and `scp` the binaries and libraries to Raspberry PI.
+
+## Cross Compile
+
+* Step 1. Install cross compile tools in the PC.
+
+``` sh
+sudo apt-get install gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
+```
+
+Or download, and install the binaries from: https://releases.linaro.org/components/toolchain/binaries/latest-7
+
+
+* Step 2. Export your experiment model to ONNX by https://github.com/wenet-e2e/wekws/blob/main/wekws/bin/export_onnx.py
+
+``` sh
+exp=exp  # Change it to your experiment dir
+python -m wekws.bin.export_onnx \
+  --config $exp/train.yaml \
+  --checkpoint $exp/final.pt \
+  --output_dir final.onnx
+```
+
+* Step 3. Build. The build requires cmake 3.14 or above. and Send the binary and libraries to Raspberry PI.
+
+``` sh
+cmake -B build -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=toolchains/aarch64-linux-gnu.toolchain.cmake
+cmake --build build
+scp -r build/bin pi@xxx.xxx.xxx:/path/to/wekws
+scp fc_base/onnxruntime-src/lib/libonnxruntime.so* pi@xxx.xxx.xxx:/path/to/wekws
+```
+
+* Step 4. Run. The log will be shown in Raspberry PI's console.
+
+``` sh
+cd /path/to/wekws
+export LD_LIBRARY_PATH=.:$LD_LIBRARY_PATH
+./build/bin/stream_kws_main 40 80 final.onnx
+```

runtime/raspberrypi/bin

@@ -0,0 +1 @@
+../core/bin

runtime/raspberrypi/cmake

@@ -0,0 +1 @@
+../core/cmake

runtime/raspberrypi/frontend

@@ -0,0 +1 @@
+../core/frontend/

runtime/raspberrypi/kws

@@ -0,0 +1 @@
+../core/kws/

runtime/raspberrypi/toolchains

@@ -0,0 +1 @@
+../core/toolchains/

runtime/raspberrypi/utils

@@ -0,0 +1 @@
+../core/utils/