wulin/qiuguo-iot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

解决新的算法生产速度已经第一句语音果断导致前端问题

Browse Source
This commit is contained in:
wulin 2023-11-21 15:18:40 +08:00
parent 037a57ea28
commit 46bcfd0184
11 changed files with 77 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/BaseSession.java
View File

@ -6,6 +6,8 @@ import lombok.Data;
import org.springframework.web.reactive.socket.WebSocketMessage;
import org.springframework.web.reactive.socket.WebSocketSession;
import reactor.core.publisher.FluxSink;
import java.util.List;
import java.util.ArrayList;
@Data
public class BaseSession {
@ -77,4 +79,9 @@ public class BaseSession {
* 用户昵称
*/
protected String nickName;
/**
* 临时记录发送给客户端resp
*/
List<BaseMessageResp> respList = new ArrayList<>(10);
}

4
iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/IActionSendMessage.java
View File

@ -22,7 +22,7 @@ public interface IActionSendMessage {
* @param baseMessageResp
* @return
*/
Mono<Boolean> sendMsgWithMono(BaseSession baseSession, BaseMessageResp baseMessageResp);
Mono<Boolean> sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp baseMessageResp);
/**
* 千问发送消息结束时调用
@ -31,7 +31,7 @@ public interface IActionSendMessage {
* @param context
* @return
*/
Mono<Boolean> sendMsgFinished(Action action, BaseSession baseSession, String context);
Mono<Boolean> sendMsgTongYiFinished(Action action, BaseSession baseSession, String context);
/**
* 获取模型Bean名称

5
iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/TTSResp.java
View File

@ -8,4 +8,9 @@ public class TTSResp {
* 1标识最后一个文本，0标识播放中
*/
Integer status = 1;
/**
* 标识顺序从0开始
*/
Integer index = 0;
}

4
iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/action/Action.java
View File

@ -69,6 +69,10 @@ public class Action {
*/
private Integer language;
/**
*
* @param v
*/
public void setNoTimeVNAsk(String v){
if(StringUtils.isNotEmpty(v)){
if(StringUtils.isNotEmpty(action)){

21
iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/ActionCommand.java
View File

@ -68,7 +68,8 @@ public abstract class ActionCommand {
QueueMessage queue,
StringBuilder sb,
Integer type,
IActionSendMessage actionSendMessage){
IActionSendMessage actionSendMessage,
Integer index){
if(baseSession.getRequestId().equals(queue.getRequestId())){
String message = "";
@ -92,22 +93,24 @@ public abstract class ActionCommand {
}
}
if(StringUtils.isNotEmpty(message)){
return normalSendMsg(baseSession, message, type, YesNo.NO.getCode(), actionSendMessage).flatMap(m -> {
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage);
return normalSendMsg(baseSession, message, type, YesNo.NO.getCode(), actionSendMessage, index).flatMap(m -> {
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, index + 1);
});
}
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage);
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, index);
}
baseSession.getRespList().clear();
return Mono.empty();
}
public Mono<Boolean> normalSendMsg(BaseSession baseSession, String message, Integer type, Integer finish, IActionSendMessage actionSendMessage){
public Mono<Boolean> normalSendMsg(BaseSession baseSession, String message, Integer type, Integer finish, IActionSendMessage actionSendMessage, Integer index){
BoxMessageResp resp = new BoxMessageResp();
resp.setType(type);
resp.setText(message);
resp.getTts().setStatus(finish);
resp.getTts().setIndex(index);
resp.setListen(baseSession.getListen());
return actionSendMessage.sendMsgWithMono(baseSession, resp);
return actionSendMessage.sendMsgWithTongYi(baseSession, resp);
}
protected Mono<Void> setQueueMessage(Action action,
@ -118,7 +121,7 @@ public abstract class ActionCommand {
return Mono.defer(() -> {
MDC.put(Log4Constans.PRINT_LOG_ID, baseSession.getLogId());
StringBuilder sb = new StringBuilder();
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage).flatMap(m -> {
return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, 0).flatMap(m -> {
MDC.remove(Log4Constans.PRINT_LOG_ID);
return Mono.empty();
});
@ -146,7 +149,9 @@ public abstract class ActionCommand {
}
return;
}
log.info("已有新的请求，不推送客户端SN：{} userId:{}", baseSession.getSn(), baseSession.getUserId());
baseSession.getRespList().clear();
}
@Override
@ -192,7 +197,7 @@ public abstract class ActionCommand {
if(data.getCode() == 200){
log.info("千问正常结束");
//保存记录
return actionSendMessage.sendMsgFinished(action, baseSession, data.getResut());
return actionSendMessage.sendMsgTongYiFinished(action, baseSession, data.getResut());
}else{
return sendMessage(action, baseSession, "很抱歉，我无法回答您的问题，请换一个问题。", AskTypeEnum.NONE.getCode(), actionSendMessage);
}

2
iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/MusicActionCommand.java
View File

@ -93,7 +93,7 @@ public class MusicActionCommand extends ActionCommand implements IActionCommand
boxSession.setMusic(resp.getMusic());
baseSession.setMusic(resp.getMusic());
}
return actionSendMessage.sendMsgWithMono(boxSession, resp);
return actionSendMessage.sendMsgWithTongYi(boxSession, resp);
}else{
log.info("设备sn：{}不在线，无法播放", baseSession.getSn());
resp.setText("设备不在线，无法播放");

2
iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/U3DActionCommand.java
View File

@ -111,7 +111,7 @@ public class U3DActionCommand extends ActionCommand implements IActionCommand{
return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(integer -> {
return toU3DMq(systemTalkBindU3d, baseSession.getUserId()).flatMap(b -> {
resp.setType(AskTypeEnum.EXIT.getCode());//告诉Box退出思考
return actionSendMessage.sendMsgWithMono(baseSession, resp);
return actionSendMessage.sendMsgWithTongYi(baseSession, resp);
});
});
}

4
iot-modules/iot-box-user-api/src/main/java/com/qiuguo/iot/user/api/controller/device/DeviceController.java
View File

@ -112,7 +112,7 @@ public class DeviceController {
}
@Override
public Mono<Boolean> sendMsgWithMono(BaseSession baseSession, BaseMessageResp resp) {
public Mono<Boolean> sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp resp) {
//BeanUtils.copyProperties(resp, baseMessageResp);
baseMessageResp.setType(resp.getType());
baseMessageResp.setText(resp.getText());
@ -120,7 +120,7 @@ public class DeviceController {
}
@Override
public Mono<Boolean> sendMsgFinished(Action action, BaseSession baseSession, String context) {
public Mono<Boolean> sendMsgTongYiFinished(Action action, BaseSession baseSession, String context) {
if(baseSession.getLanguage().equals(LanguageEnum.ENGLISH.getCode())){
baseMessageResp.setText(aliYunTranslateService.ChineseToEnglish(context));
}else{

2
iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java
View File

@ -176,7 +176,7 @@ public class BoxWebSocketHandler implements WebSocketHandler {
resp.setText(alarm.getReadText());
resp.setListen(0);
resp.setType(AskTypeEnum.TTS.getCode());
return baseWebSocketService.sendMsgWithMono(boxSession, resp).flatMap(b -> {
return baseWebSocketService.sendMsgWithTongYi(boxSession, resp).flatMap(b -> {
return Mono.empty();
});
}else{

74
iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/service/BaseWebSocketService.java
View File

@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.qiuguo.iot.base.constans.RedisConstans;
import com.qiuguo.iot.base.enums.AskTypeEnum;
import com.qiuguo.iot.base.enums.LanguageEnum;
import com.qiuguo.iot.base.enums.YesNo;
import com.qiuguo.iot.base.utils.Md5Utils;
import com.qiuguo.iot.base.utils.StringUtils;
import com.qiuguo.iot.box.websocket.api.config.properties.SomeProperties;
@ -68,6 +69,12 @@ public class BaseWebSocketService implements IActionSendMessage {
@Autowired
private AliYunTranslateService aliYunTranslateService;
/**
* 推送给前端时，当有多条有可能音频前面的音频时间很短导致。音频播放完了还没有推送下一条音频。从而导致无法继续播放
* 这里用来记录内存中最多保存多少条，达到该条后就一次性把内存中的推送给前端。后面的继续一条条的推送
*/
private static int MAX_INNER_RESP = 2;
@ -192,14 +199,14 @@ public class BaseWebSocketService implements IActionSendMessage {
}
@Override
public Mono<Boolean> sendMsgFinished(Action action, BaseSession baseSession, String context){
public Mono<Boolean> sendMsgTongYiFinished(Action action, BaseSession baseSession, String context){
return saveTalkRecord(baseSession, action, context).flatMap(i -> {
return Mono.just(true);
});
}
@Override
public Mono<Boolean> sendMsgWithMono(BaseSession baseSession, BaseMessageResp baseMessageResp) {
public Mono<Boolean> sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp baseMessageResp) {
if(baseSession instanceof BoxSession){
log.info("果box聊天记录，同步到客户端");
BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId());
@ -348,16 +355,45 @@ public class BaseWebSocketService implements IActionSendMessage {
if(baseSession.getLanguage().equals(LanguageEnum.ENGLISH.getCode())){
boxMessageResp.setText(aliYunTranslateService.ChineseToEnglish(boxMessageResp.getText()));
}
return audioService.getAudioUrl(boxMessageResp.getText(),
return audioService.getAudio20231117Url(boxMessageResp.getText(),
StringUtils.isNotEmpty(baseSession.getTtsId()) ? baseSession.getTtsId() : "scl"
).map(s ->{
boxMessageResp.setAudio(s);
if(boxMessageResp.getTts().getStatus().equals(YesNo.NO.getCode())){
//多条的第一句
if(boxMessageResp.getTts().getIndex().intValue() < MAX_INNER_RESP){
baseSession.getRespList().add(boxMessageResp);
return s;
}else if(boxMessageResp.getTts().getIndex().intValue() == MAX_INNER_RESP){
//不一定会执行到
sendInnerMsg(baseSession);
baseSession.getRespList().clear();
sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp));
return s;
}
}else{
if(baseSession.getRespList().size() > 0){
sendInnerMsg(baseSession);
}
}
sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp));
return s;
});//.subscribeOn(Schedulers.single()).subscribe();
}
private void sendInnerMsg(BaseSession baseSession){
for(BaseMessageResp response : baseSession.getRespList()){
try{
sendMsg(baseSession, JSONObject.toJSONString(response));
Thread.sleep(10);
}catch(Exception e){
log.info("推送内存中消息给客户端时异常{}", e);
}
}
baseSession.getRespList().clear();
}
public Mono<DeviceInfoEntity> checkBoxSignature(String sn, String signature, Long linkTime){
return reactiveStringRedisTemplate.opsForValue()
.get(RedisConstans.DEVICE_INFO + sn).defaultIfEmpty("")
@ -390,34 +426,4 @@ public class BaseWebSocketService implements IActionSendMessage {
});
}
/*public static void main(String[] args) {
String v = "我来讲一个英文故事给你听吧！Once upon a time, there was a little girl named Goldilocks. She went for a walk in the forest. Pretty soon, she came upon a house. She knocked and, when no one answered, she walked right in. At the table in the kitchen, there were three bowls of porridge. Goldilocks was hungry. She tasted the porridge from the first bowl. \"This porridge is too hot!\"she exclaimed. So, she tasted the porridge from the second bowl. \"This porridge is too cold,\"she said. So, she tasted the last bowl of porridge. \"Ahhh, this porridge is just right!\"she exclaimed happily. \"What a lovely house this is!\"she said to herself. She looked into the living room. There were chairs, a couch, and a rocking chair. Goldilocks walked over to the rocking chair. It was too big. Then she saw a chair that looked just right. She sat down in it and it was just right. Next, she saw a bed. The first bed she saw was too hard. The next bed was too soft. But the last bed was just right. Goldilocks lay down and went fast asleep. Suddenly, she heard a great noise downstairs. Someone was coming! She jumped out of bed, ran into the bathroom, and hid behind the door. Three bears entered the house. They came into the kitchen. \"Someone has been eating my porridge,\"said the first bear. \"Someone has been eating my porridge,\"said the second bear. \"Someone has been eating my porridge,\"said the third bear. \"And someone has been sitting in my chair.\"\"And someone has been sitting in my chair.\"\"And someone has been sitting in my chair.\"\"And someone has been sleeping in my bed.\"\"And someone has been sleeping in my bed.\"\"And someone has been sleeping in my bed.\"They followed the trail of footprints upstairs. When they got to the bedroom, Goldilocks heard them say, \"Someone has been sleeping in my bed.\"Goldilocks trembled with fear. She opened the bathroom door very slowly and peeked out. She saw three angry bears looking at her. She closed the door quickly and hid behind it. The bears broke down the door, and Goldilocks ran out of the house as fast as she could. She ran so fast that she fell down and broke all the bones in her body. And that is the end of the story of Goldilocks and the Three Bears.";
StringBuilder sb = new StringBuilder();
pro(v, 0, v.length(), sb, 50);
}
public static void pro(String text, int n, int length, StringBuilder builder, int ONE_MAX_TEXT){
if(n < length){
int start = n;
n += ONE_MAX_TEXT;
if(n > length){
n = length;
}
String message = text.substring(start, n);
int status = 0;
if(n == length){
status = 1;
}
message = StringUtils.getSendStr(builder, message, status == 1 ? true : false);
int m = n;
if(StringUtils.isNotEmpty(message)){
System.out.println(message);
pro(text, n, length, builder, ONE_MAX_TEXT);
}else if(n < length){
pro(text, n, length, builder, ONE_MAX_TEXT);
}
}
}*/
}

2
iot-modules/iot-box-websocket-api/src/main/resources/bootstrap-dev.yml
View File

@ -51,7 +51,7 @@ tts:
#max:
# text: 30 #当发送给前端文字超过改值时，分批发送给前端
suanfa: true #nacos控制变化
url: http://47.110.40.30:18000/run/predict #http://192.168.8.211:18000/run/predict #http://47.110.40.30:18000/run/predict # #算法语音合成
url: http://192.168.8.211:18000/run/predict # #http://47.110.40.30:18000/run/predict # #算法语音合成
lac:
#type: suanfa #nacos控制变化
huburl: http://192.168.8.175:8866/predict/lac