diff --git a/runtime/android/.gitignore b/runtime/android/.gitignore new file mode 100644 index 0000000..aa724b7 --- /dev/null +++ b/runtime/android/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/runtime/android/app/.gitignore b/runtime/android/app/.gitignore new file mode 100644 index 0000000..42afabf --- /dev/null +++ b/runtime/android/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/runtime/android/app/proguard-rules.pro b/runtime/android/app/proguard-rules.pro new file mode 100644 index 0000000..481bb43 --- /dev/null +++ b/runtime/android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/runtime/android/app/src/androidTest/java/cn/org/wenet/wekws/ExampleInstrumentedTest.java b/runtime/android/app/src/androidTest/java/cn/org/wenet/wekws/ExampleInstrumentedTest.java new file mode 100644 index 0000000..7aa4c52 --- /dev/null +++ b/runtime/android/app/src/androidTest/java/cn/org/wenet/wekws/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package cn.org.wenet.wekws; + +import android.content.Context; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.ext.junit.runners.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); + assertEquals("cn.org.wenet.wekws", appContext.getPackageName()); + } +} \ No newline at end of file diff --git a/runtime/android/app/src/main/AndroidManifest.xml b/runtime/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..8f25ec3 --- /dev/null +++ b/runtime/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/android/app/src/main/cpp/CMakeLists.txt b/runtime/android/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..589d530 --- /dev/null +++ b/runtime/android/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.4.1) +project(wekws CXX) +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_VERBOSE_MAKEFILE on) + +set(build_DIR ${CMAKE_SOURCE_DIR}/../../../build) +file(GLOB ONNXRUNTIME_INCLUDE_DIRS "${build_DIR}/onnxruntime*.aar/headers") +file(GLOB ONNXRUNTIME_LINK_DIRS "${build_DIR}/onnxruntime*.aar/jni/${ANDROID_ABI}") +link_directories(${ONNXRUNTIME_LINK_DIRS}) +include_directories(${ONNXRUNTIME_INCLUDE_DIRS}) + +include_directories(${CMAKE_SOURCE_DIR}) +add_library(wekws SHARED + frontend/feature_pipeline.cc + frontend/fft.cc + kws/keyword_spotting.cc + wekws.cc +) +target_link_libraries(wekws PUBLIC onnxruntime) diff --git a/runtime/android/app/src/main/cpp/frontend b/runtime/android/app/src/main/cpp/frontend new file mode 120000 index 0000000..22be6cd --- /dev/null +++ b/runtime/android/app/src/main/cpp/frontend @@ -0,0 +1 @@ +../../../../../core/frontend \ No newline at end of file diff --git a/runtime/android/app/src/main/cpp/kws b/runtime/android/app/src/main/cpp/kws new file mode 120000 index 0000000..b2689b9 --- /dev/null +++ b/runtime/android/app/src/main/cpp/kws @@ -0,0 +1 @@ +../../../../../core/kws \ No newline at end of file diff --git a/runtime/android/app/src/main/cpp/utils b/runtime/android/app/src/main/cpp/utils new file mode 120000 index 0000000..2530bac --- /dev/null +++ b/runtime/android/app/src/main/cpp/utils @@ -0,0 +1 @@ +../../../../../core/utils \ No newline at end of file diff --git a/runtime/android/app/src/main/cpp/wekws.cc b/runtime/android/app/src/main/cpp/wekws.cc new file mode 100644 index 0000000..d7fe24f --- /dev/null +++ b/runtime/android/app/src/main/cpp/wekws.cc @@ -0,0 +1,119 @@ +// Copyright (c) 2022 Zhendong Peng (pzd17@tsinghua.org.cn) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include + +#include +#include + +#include "frontend/feature_pipeline.h" +#include "kws/keyword_spotting.h" +#include "utils/log.h" + +namespace wekws { +std::shared_ptr spotter; +std::shared_ptr feature_config; +std::shared_ptr feature_pipeline; +std::string result; // NOLINT +int offset; + +void init(JNIEnv* env, jobject, jstring jModelDir) { + const char* pModelDir = env->GetStringUTFChars(jModelDir, nullptr); + + std::string modelPath = std::string(pModelDir) + "/wenwen.ort"; + spotter = std::make_shared(modelPath); + + feature_config = std::make_shared(40, 16000); + feature_pipeline = std::make_shared(*feature_config); +} + +void reset(JNIEnv *env, jobject) { + offset = 0; + result = ""; + spotter->Reset(); +} + +void accept_waveform(JNIEnv *env, jobject, jshortArray jWaveform) { + jsize size = env->GetArrayLength(jWaveform); + int16_t* waveform = env->GetShortArrayElements(jWaveform, 0); + std::vector v(waveform, waveform + size); + feature_pipeline->AcceptWaveform(v); + LOG(INFO) << "wekws accept waveform in ms: " << int(size / 16); +} + +void set_input_finished() { + LOG(INFO) << "wekws input finished"; + feature_pipeline->set_input_finished(); +} + +void spot_thread_func() { + while (true) { + std::vector> feats; + feature_pipeline->Read(80, &feats); + std::vector> prob; + spotter->Forward(feats, &prob); + + float max_hi_xiaowen = 0; + float max_nihao_wenwen = 0; + for (int t = 0; t < prob.size(); t++) { + max_hi_xiaowen = std::max(prob[t][0], max_hi_xiaowen); + max_nihao_wenwen = std::max(prob[t][1], max_nihao_wenwen); + } + float max_prob = max_hi_xiaowen + max_nihao_wenwen; + result = std::to_string(offset) + " prob: " + std::to_string(max_prob); + offset += prob.size(); + } +} + +void start_spot() { + std::thread decode_thread(spot_thread_func); + decode_thread.detach(); +} + +jstring get_result(JNIEnv *env, jobject) { + LOG(INFO) << "wekws ui result: " << result; + return env->NewStringUTF(result.c_str()); +} +} // namespace wekws + +JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *) { + JNIEnv *env; + if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { + return JNI_ERR; + } + + jclass c = env->FindClass("cn/org/wenet/wekws/Spot"); + if (c == nullptr) { + return JNI_ERR; + } + + static const JNINativeMethod methods[] = { + {"init", "(Ljava/lang/String;)V", reinterpret_cast(wekws::init)}, + {"reset", "()V", reinterpret_cast(wekws::reset)}, + {"acceptWaveform", "([S)V", + reinterpret_cast(wekws::accept_waveform)}, + {"setInputFinished", "()V", + reinterpret_cast(wekws::set_input_finished)}, + {"startSpot", "()V", reinterpret_cast(wekws::start_spot)}, + {"getResult", "()Ljava/lang/String;", + reinterpret_cast(wekws::get_result)}, + }; + int rc = env->RegisterNatives(c, methods, + sizeof(methods) / sizeof(JNINativeMethod)); + + if (rc != JNI_OK) { + return rc; + } + + return JNI_VERSION_1_6; +} diff --git a/runtime/android/app/src/main/java/cn/org/wenet/wekws/MainActivity.java b/runtime/android/app/src/main/java/cn/org/wenet/wekws/MainActivity.java new file mode 100644 index 0000000..e1b0d08 --- /dev/null +++ b/runtime/android/app/src/main/java/cn/org/wenet/wekws/MainActivity.java @@ -0,0 +1,212 @@ +package cn.org.wenet.wekws; + +import androidx.appcompat.app.AppCompatActivity; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +import android.Manifest; +import android.content.Context; +import android.content.pm.PackageManager; +import android.content.res.AssetManager; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder; +import android.os.Bundle; +import android.os.Process; +import android.util.Log; +import android.widget.Button; +import android.widget.TextView; +import android.widget.Toast; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +public class MainActivity extends AppCompatActivity { + + private final int MY_PERMISSIONS_RECORD_AUDIO = 1; + private static final String LOG_TAG = "WEKWS"; + private static final int SAMPLE_RATE = 16000; // The sampling rate + private static final int MAX_QUEUE_SIZE = 2500; // 100 seconds audio, 1 / 0.04 * 100 + private static final List resource = Arrays.asList("wenwen.ort"); + + private boolean startRecord = false; + private AudioRecord record = null; + private int miniBufferSize = 0; // 1280 bytes 648 byte 40ms, 0.04s + private final BlockingQueue bufferQueue = new ArrayBlockingQueue<>(MAX_QUEUE_SIZE); + + public static void assetsInit(Context context) throws IOException { + AssetManager assetMgr = context.getAssets(); + // Unzip all files in resource from assets to context. + // Note: Uninstall the APP will remove the resource files in the context. + for (String file : assetMgr.list("")) { + if (resource.contains(file)) { + File dst = new File(context.getFilesDir(), file); + if (!dst.exists() || dst.length() == 0) { + Log.i(LOG_TAG, "Unzipping " + file + " to " + dst.getAbsolutePath()); + InputStream is = assetMgr.open(file); + OutputStream os = new FileOutputStream(dst); + byte[] buffer = new byte[4 * 1024]; + int read; + while ((read = is.read(buffer)) != -1) { + os.write(buffer, 0, read); + } + os.flush(); + } + } + } + } + + @Override + public void onRequestPermissionsResult(int requestCode, + String[] permissions, int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (requestCode == MY_PERMISSIONS_RECORD_AUDIO) { + if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED) { + Log.i(LOG_TAG, "record permission is granted"); + initRecorder(); + } else { + Toast.makeText(this, "Permissions denied to record audio", Toast.LENGTH_LONG).show(); + Button button = findViewById(R.id.button); + button.setEnabled(false); + } + } + } + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + requestAudioPermissions(); + try { + assetsInit(this); + } catch (IOException e) { + Log.e(LOG_TAG, "Error process asset files to file path"); + } + + TextView textView = findViewById(R.id.textView); + textView.setText(""); + Spot.init(getFilesDir().getPath()); + + Button button = findViewById(R.id.button); + button.setText("Start Record"); + button.setOnClickListener(view -> { + if (!startRecord) { + startRecord = true; + startRecordThread(); + startSpotThread(); + Spot.reset(); + Spot.startSpot(); + button.setText("Stop Record"); + } else { + startRecord = false; + button.setText("Start Record"); + } + }); + } + + private void requestAudioPermissions() { + if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) + != PackageManager.PERMISSION_GRANTED) { + ActivityCompat.requestPermissions(this, + new String[]{Manifest.permission.RECORD_AUDIO}, + MY_PERMISSIONS_RECORD_AUDIO); + } else { + initRecorder(); + } + } + + private void initRecorder() { + // buffer size in bytes 1280 + miniBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT); + if (miniBufferSize == AudioRecord.ERROR || miniBufferSize == AudioRecord.ERROR_BAD_VALUE) { + Log.e(LOG_TAG, "Audio buffer can't initialize!"); + return; + } + if (ActivityCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + // TODO: Consider calling + // ActivityCompat#requestPermissions + // here to request the missing permissions, and then overriding + // public void onRequestPermissionsResult(int requestCode, String[] permissions, + // int[] grantResults) + // to handle the case where the user grants the permission. See the documentation + // for ActivityCompat#requestPermissions for more details. + return; + } + record = new AudioRecord(MediaRecorder.AudioSource.DEFAULT, + SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + miniBufferSize); + if (record.getState() != AudioRecord.STATE_INITIALIZED) { + Log.e(LOG_TAG, "Audio Record can't initialize!"); + return; + } + Log.i(LOG_TAG, "Record init okay"); + } + + private void startRecordThread() { + new Thread(() -> { + VoiceRectView voiceView = findViewById(R.id.voiceRectView); + record.startRecording(); + Process.setThreadPriority(Process.THREAD_PRIORITY_AUDIO); + while (startRecord) { + short[] buffer = new short[miniBufferSize / 2]; + int read = record.read(buffer, 0, buffer.length); + voiceView.add(calculateDb(buffer)); + try { + if (AudioRecord.ERROR_INVALID_OPERATION != read) { + bufferQueue.put(buffer); + } + } catch (InterruptedException e) { + Log.e(LOG_TAG, e.getMessage()); + } + Button button = findViewById(R.id.button); + if (!button.isEnabled() && startRecord) { + runOnUiThread(() -> button.setEnabled(true)); + } + } + record.stop(); + voiceView.zero(); + }).start(); + } + + private double calculateDb(short[] buffer) { + double energy = 0.0; + for (short value : buffer) { + energy += value * value; + } + energy /= buffer.length; + energy = (10 * Math.log10(1 + energy)) / 100; + energy = Math.min(energy, 1.0); + return energy; + } + + private void startSpotThread() { + new Thread(() -> { + // Send all data + while (startRecord || bufferQueue.size() > 0) { + try { + short[] data = bufferQueue.take(); + // 1. add data to C++ interface + Spot.acceptWaveform(data); + // 2. get partial result + runOnUiThread(() -> { + TextView textView = findViewById(R.id.textView); + textView.setText(Spot.getResult()); + }); + } catch (InterruptedException e) { + Log.e(LOG_TAG, e.getMessage()); + } + } + }).start(); + } +} \ No newline at end of file diff --git a/runtime/android/app/src/main/java/cn/org/wenet/wekws/Spot.java b/runtime/android/app/src/main/java/cn/org/wenet/wekws/Spot.java new file mode 100644 index 0000000..fc38ec8 --- /dev/null +++ b/runtime/android/app/src/main/java/cn/org/wenet/wekws/Spot.java @@ -0,0 +1,15 @@ +package cn.org.wenet.wekws; + +public class Spot { + + static { + System.loadLibrary("wekws"); + } + + public static native void init(String modelDir); + public static native void reset(); + public static native void acceptWaveform(short[] waveform); + public static native void setInputFinished(); + public static native void startSpot(); + public static native String getResult(); +} diff --git a/runtime/android/app/src/main/java/cn/org/wenet/wekws/VoiceRectView.java b/runtime/android/app/src/main/java/cn/org/wenet/wekws/VoiceRectView.java new file mode 100644 index 0000000..8d3736b --- /dev/null +++ b/runtime/android/app/src/main/java/cn/org/wenet/wekws/VoiceRectView.java @@ -0,0 +1,136 @@ +package cn.org.wenet.wekws; + +import android.content.Context; +import android.content.res.TypedArray; +import android.graphics.Canvas; +import android.graphics.LinearGradient; +import android.graphics.Paint; +import android.graphics.Shader; +import android.util.AttributeSet; +import android.view.View; + +import androidx.core.content.ContextCompat; + +import java.util.Arrays; + +/** + * 自定义的音频模拟条形图 Created by shize on 2016/9/5. + */ +public class VoiceRectView extends View { + + // 音频矩形的数量 + private int mRectCount; + // 音频矩形的画笔 + private Paint mRectPaint; + // 渐变颜色的两种 + private int topColor, downColor; + // 音频矩形的宽和高 + private int mRectWidth, mRectHeight; + // 偏移量 + private int offset; + // 频率速度 + private int mSpeed; + + private double[] mEnergyBuffer = null; + + public VoiceRectView(Context context) { + this(context, null); + } + + public VoiceRectView(Context context, AttributeSet attrs) { + this(context, attrs, 0); + } + + public VoiceRectView(Context context, AttributeSet attrs, int defStyleAttr) { + super(context, attrs, defStyleAttr); + setPaint(context, attrs); + } + + public void setPaint(Context context, AttributeSet attrs) { + // 将属性存储到TypedArray中 + TypedArray ta = context.obtainStyledAttributes(attrs, R.styleable.VoiceRect); + mRectPaint = new Paint(); + // 添加矩形画笔的基础颜色 + mRectPaint.setColor(ta.getColor(R.styleable.VoiceRect_RectTopColor, + ContextCompat.getColor(context, R.color.top_color))); + // 添加矩形渐变色的上面部分 + topColor = ta.getColor(R.styleable.VoiceRect_RectTopColor, + ContextCompat.getColor(context, R.color.top_color)); + // 添加矩形渐变色的下面部分 + downColor = ta.getColor(R.styleable.VoiceRect_RectDownColor, + ContextCompat.getColor(context, R.color.down_color)); + // 设置矩形的数量 + mRectCount = ta.getInt(R.styleable.VoiceRect_RectCount, 10); + mEnergyBuffer = new double[mRectCount]; + + // 设置重绘的时间间隔，也就是变化速度 + mSpeed = ta.getInt(R.styleable.VoiceRect_RectSpeed, 300); + // 每个矩形的间隔 + offset = ta.getInt(R.styleable.VoiceRect_RectOffset, 0); + // 回收TypeArray + ta.recycle(); + } + + @Override + protected void onSizeChanged(int w, int h, int oldW, int oldH) { + super.onSizeChanged(w, h, oldW, oldH); + // 渐变效果 + LinearGradient mLinearGradient; + // 画布的宽 + int mWidth; + // 获取画布的宽 + mWidth = getWidth(); + // 获取矩形的最大高度 + mRectHeight = getHeight(); + // 获取单个矩形的宽度(减去的部分为到右边界的间距) + mRectWidth = (mWidth - offset) / mRectCount; + // 实例化一个线性渐变 + mLinearGradient = new LinearGradient( + 0, + 0, + mRectWidth, + mRectHeight, + topColor, + downColor, + Shader.TileMode.CLAMP + ); + // 添加进画笔的着色器 + mRectPaint.setShader(mLinearGradient); + } + + public void add(double energy) { + if (mEnergyBuffer.length - 1 >= 0) { + System.arraycopy(mEnergyBuffer, 1, mEnergyBuffer, 0, mEnergyBuffer.length - 1); + } + mEnergyBuffer[mEnergyBuffer.length - 1] = energy; + } + + public void zero() { + Arrays.fill(mEnergyBuffer, 0); + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + double mRandom; + float currentHeight; + for (int i = 0; i < mRectCount; i++) { + // 由于只是简单的案例就不监听音频输入，随机模拟一些数字即可 + mRandom = Math.random(); + + //if (i < 1 || i > mRectCount - 2) mRandom = 0; + currentHeight = (float) (mRectHeight * mEnergyBuffer[i]); + + // 矩形的绘制是从左边开始到上、右、下边（左右边距离左边画布边界的距离，上下边距离上边画布边界的距离） + canvas.drawRect( + (float) (mRectWidth * i + offset), + (mRectHeight - currentHeight) / 2, + (float) (mRectWidth * (i + 1)), + mRectHeight / 2 + currentHeight / 2, + mRectPaint + ); + } + // 使得view延迟重绘 + postInvalidateDelayed(mSpeed); + } +} \ No newline at end of file diff --git a/runtime/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/runtime/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 0000000..2b068d1 --- /dev/null +++ b/runtime/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/runtime/android/app/src/main/res/drawable/ic_launcher_background.xml b/runtime/android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000..07d5da9 --- /dev/null +++ b/runtime/android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/runtime/android/app/src/main/res/layout/activity_main.xml b/runtime/android/app/src/main/res/layout/activity_main.xml new file mode 100644 index 0000000..c61b7d0 --- /dev/null +++ b/runtime/android/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,50 @@ + + + + + + + +