diff --git a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/BaseSession.java b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/BaseSession.java index 219a705..b19374c 100644 --- a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/BaseSession.java +++ b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/BaseSession.java @@ -6,6 +6,8 @@ import lombok.Data; import org.springframework.web.reactive.socket.WebSocketMessage; import org.springframework.web.reactive.socket.WebSocketSession; import reactor.core.publisher.FluxSink; +import java.util.List; +import java.util.ArrayList; @Data public class BaseSession { @@ -77,4 +79,9 @@ public class BaseSession { * 用户昵称 */ protected String nickName; + + /** + * 临时记录发送给客户端resp + */ + List respList = new ArrayList<>(10); } diff --git a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/IActionSendMessage.java b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/IActionSendMessage.java index 26dcf94..fd5e0f7 100644 --- a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/IActionSendMessage.java +++ b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/IActionSendMessage.java @@ -22,7 +22,7 @@ public interface IActionSendMessage { * @param baseMessageResp * @return */ - Mono sendMsgWithMono(BaseSession baseSession, BaseMessageResp baseMessageResp); + Mono sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp baseMessageResp); /** * 千问发送消息结束时调用 @@ -31,7 +31,7 @@ public interface IActionSendMessage { * @param context * @return */ - Mono sendMsgFinished(Action action, BaseSession baseSession, String context); + Mono sendMsgTongYiFinished(Action action, BaseSession baseSession, String context); /** * 获取模型Bean名称 diff --git a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/TTSResp.java b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/TTSResp.java index a161126..37b8856 100644 --- a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/TTSResp.java +++ b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/TTSResp.java @@ -8,4 +8,9 @@ public class TTSResp { * 1标识最后一个文本，0标识播放中 */ Integer status = 1; + + /** + * 标识顺序从0开始 + */ + Integer index = 0; } diff --git a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/action/Action.java b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/action/Action.java index edaa606..a9eb899 100644 --- a/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/action/Action.java +++ b/iot-common/iot-data/src/main/java/com/qiuguo/iot/data/domain/action/Action.java @@ -69,6 +69,10 @@ public class Action { */ private Integer language; + /** + * + * @param v + */ public void setNoTimeVNAsk(String v){ if(StringUtils.isNotEmpty(v)){ if(StringUtils.isNotEmpty(action)){ diff --git a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/ActionCommand.java b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/ActionCommand.java index 8e4bef1..e169540 100644 --- a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/ActionCommand.java +++ b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/ActionCommand.java @@ -68,7 +68,8 @@ public abstract class ActionCommand { QueueMessage queue, StringBuilder sb, Integer type, - IActionSendMessage actionSendMessage){ + IActionSendMessage actionSendMessage, + Integer index){ if(baseSession.getRequestId().equals(queue.getRequestId())){ String message = ""; @@ -92,22 +93,24 @@ public abstract class ActionCommand { } } if(StringUtils.isNotEmpty(message)){ - return normalSendMsg(baseSession, message, type, YesNo.NO.getCode(), actionSendMessage).flatMap(m -> { - return sendMessage(action, baseSession, queue, sb, type, actionSendMessage); + return normalSendMsg(baseSession, message, type, YesNo.NO.getCode(), actionSendMessage, index).flatMap(m -> { + return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, index + 1); }); } - return sendMessage(action, baseSession, queue, sb, type, actionSendMessage); + return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, index); } + baseSession.getRespList().clear(); return Mono.empty(); } - public Mono normalSendMsg(BaseSession baseSession, String message, Integer type, Integer finish, IActionSendMessage actionSendMessage){ + public Mono normalSendMsg(BaseSession baseSession, String message, Integer type, Integer finish, IActionSendMessage actionSendMessage, Integer index){ BoxMessageResp resp = new BoxMessageResp(); resp.setType(type); resp.setText(message); resp.getTts().setStatus(finish); + resp.getTts().setIndex(index); resp.setListen(baseSession.getListen()); - return actionSendMessage.sendMsgWithMono(baseSession, resp); + return actionSendMessage.sendMsgWithTongYi(baseSession, resp); } protected Mono setQueueMessage(Action action, @@ -118,7 +121,7 @@ public abstract class ActionCommand { return Mono.defer(() -> { MDC.put(Log4Constans.PRINT_LOG_ID, baseSession.getLogId()); StringBuilder sb = new StringBuilder(); - return sendMessage(action, baseSession, queue, sb, type, actionSendMessage).flatMap(m -> { + return sendMessage(action, baseSession, queue, sb, type, actionSendMessage, 0).flatMap(m -> { MDC.remove(Log4Constans.PRINT_LOG_ID); return Mono.empty(); }); @@ -146,7 +149,9 @@ public abstract class ActionCommand { } return; } + log.info("已有新的请求，不推送客户端SN：{} userId:{}", baseSession.getSn(), baseSession.getUserId()); + baseSession.getRespList().clear(); } @Override @@ -192,7 +197,7 @@ public abstract class ActionCommand { if(data.getCode() == 200){ log.info("千问正常结束"); //保存记录 - return actionSendMessage.sendMsgFinished(action, baseSession, data.getResut()); + return actionSendMessage.sendMsgTongYiFinished(action, baseSession, data.getResut()); }else{ return sendMessage(action, baseSession, "很抱歉，我无法回答您的问题，请换一个问题。", AskTypeEnum.NONE.getCode(), actionSendMessage); } diff --git a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/MusicActionCommand.java b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/MusicActionCommand.java index 3896a93..cef84a7 100644 --- a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/MusicActionCommand.java +++ b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/MusicActionCommand.java @@ -93,7 +93,7 @@ public class MusicActionCommand extends ActionCommand implements IActionCommand boxSession.setMusic(resp.getMusic()); baseSession.setMusic(resp.getMusic()); } - return actionSendMessage.sendMsgWithMono(boxSession, resp); + return actionSendMessage.sendMsgWithTongYi(boxSession, resp); }else{ log.info("设备sn：{}不在线，无法播放", baseSession.getSn()); resp.setText("设备不在线，无法播放"); diff --git a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/U3DActionCommand.java b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/U3DActionCommand.java index 45e292c..9df5ee4 100644 --- a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/U3DActionCommand.java +++ b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/command/U3DActionCommand.java @@ -111,7 +111,7 @@ public class U3DActionCommand extends ActionCommand implements IActionCommand{ return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(integer -> { return toU3DMq(systemTalkBindU3d, baseSession.getUserId()).flatMap(b -> { resp.setType(AskTypeEnum.EXIT.getCode());//告诉Box退出思考 - return actionSendMessage.sendMsgWithMono(baseSession, resp); + return actionSendMessage.sendMsgWithTongYi(baseSession, resp); }); }); } diff --git a/iot-modules/iot-box-user-api/src/main/java/com/qiuguo/iot/user/api/controller/device/DeviceController.java b/iot-modules/iot-box-user-api/src/main/java/com/qiuguo/iot/user/api/controller/device/DeviceController.java index b0033ad..36f06c9 100644 --- a/iot-modules/iot-box-user-api/src/main/java/com/qiuguo/iot/user/api/controller/device/DeviceController.java +++ b/iot-modules/iot-box-user-api/src/main/java/com/qiuguo/iot/user/api/controller/device/DeviceController.java @@ -112,7 +112,7 @@ public class DeviceController { } @Override - public Mono sendMsgWithMono(BaseSession baseSession, BaseMessageResp resp) { + public Mono sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp resp) { //BeanUtils.copyProperties(resp, baseMessageResp); baseMessageResp.setType(resp.getType()); baseMessageResp.setText(resp.getText()); @@ -120,7 +120,7 @@ public class DeviceController { } @Override - public Mono sendMsgFinished(Action action, BaseSession baseSession, String context) { + public Mono sendMsgTongYiFinished(Action action, BaseSession baseSession, String context) { if(baseSession.getLanguage().equals(LanguageEnum.ENGLISH.getCode())){ baseMessageResp.setText(aliYunTranslateService.ChineseToEnglish(context)); }else{ diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java index 9f5f245..a2c6391 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java @@ -176,7 +176,7 @@ public class BoxWebSocketHandler implements WebSocketHandler { resp.setText(alarm.getReadText()); resp.setListen(0); resp.setType(AskTypeEnum.TTS.getCode()); - return baseWebSocketService.sendMsgWithMono(boxSession, resp).flatMap(b -> { + return baseWebSocketService.sendMsgWithTongYi(boxSession, resp).flatMap(b -> { return Mono.empty(); }); }else{ diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/service/BaseWebSocketService.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/service/BaseWebSocketService.java index 9091c27..1c1d5a1 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/service/BaseWebSocketService.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/service/BaseWebSocketService.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject; import com.qiuguo.iot.base.constans.RedisConstans; import com.qiuguo.iot.base.enums.AskTypeEnum; import com.qiuguo.iot.base.enums.LanguageEnum; +import com.qiuguo.iot.base.enums.YesNo; import com.qiuguo.iot.base.utils.Md5Utils; import com.qiuguo.iot.base.utils.StringUtils; import com.qiuguo.iot.box.websocket.api.config.properties.SomeProperties; @@ -68,6 +69,12 @@ public class BaseWebSocketService implements IActionSendMessage { @Autowired private AliYunTranslateService aliYunTranslateService; + /** + * 推送给前端时，当有多条有可能音频前面的音频时间很短导致。音频播放完了还没有推送下一条音频。从而导致无法继续播放 + * 这里用来记录内存中最多保存多少条，达到该条后就一次性把内存中的推送给前端。后面的继续一条条的推送 + */ + + private static int MAX_INNER_RESP = 2; @@ -192,14 +199,14 @@ public class BaseWebSocketService implements IActionSendMessage { } @Override - public Mono sendMsgFinished(Action action, BaseSession baseSession, String context){ + public Mono sendMsgTongYiFinished(Action action, BaseSession baseSession, String context){ return saveTalkRecord(baseSession, action, context).flatMap(i -> { return Mono.just(true); }); } @Override - public Mono sendMsgWithMono(BaseSession baseSession, BaseMessageResp baseMessageResp) { + public Mono sendMsgWithTongYi(BaseSession baseSession, BaseMessageResp baseMessageResp) { if(baseSession instanceof BoxSession){ log.info("果box聊天记录，同步到客户端"); BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId()); @@ -348,16 +355,45 @@ public class BaseWebSocketService implements IActionSendMessage { if(baseSession.getLanguage().equals(LanguageEnum.ENGLISH.getCode())){ boxMessageResp.setText(aliYunTranslateService.ChineseToEnglish(boxMessageResp.getText())); } - return audioService.getAudioUrl(boxMessageResp.getText(), + return audioService.getAudio20231117Url(boxMessageResp.getText(), StringUtils.isNotEmpty(baseSession.getTtsId()) ? baseSession.getTtsId() : "scl" ).map(s ->{ - boxMessageResp.setAudio(s); + if(boxMessageResp.getTts().getStatus().equals(YesNo.NO.getCode())){ + //多条的第一句 + if(boxMessageResp.getTts().getIndex().intValue() < MAX_INNER_RESP){ + baseSession.getRespList().add(boxMessageResp); + return s; + }else if(boxMessageResp.getTts().getIndex().intValue() == MAX_INNER_RESP){ + //不一定会执行到 + sendInnerMsg(baseSession); + baseSession.getRespList().clear(); + sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); + return s; + } + }else{ + if(baseSession.getRespList().size() > 0){ + sendInnerMsg(baseSession); + } + } sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); return s; });//.subscribeOn(Schedulers.single()).subscribe(); } + private void sendInnerMsg(BaseSession baseSession){ + for(BaseMessageResp response : baseSession.getRespList()){ + + try{ + sendMsg(baseSession, JSONObject.toJSONString(response)); + Thread.sleep(10); + }catch(Exception e){ + log.info("推送内存中消息给客户端时异常{}", e); + } + } + baseSession.getRespList().clear(); + } + public Mono checkBoxSignature(String sn, String signature, Long linkTime){ return reactiveStringRedisTemplate.opsForValue() .get(RedisConstans.DEVICE_INFO + sn).defaultIfEmpty("") @@ -390,34 +426,4 @@ public class BaseWebSocketService implements IActionSendMessage { }); } - /*public static void main(String[] args) { - String v = "我来讲一个英文故事给你听吧！Once upon a time, there was a little girl named Goldilocks. She went for a walk in the forest. Pretty soon, she came upon a house. She knocked and, when no one answered, she walked right in. At the table in the kitchen, there were three bowls of porridge. Goldilocks was hungry. She tasted the porridge from the first bowl. \"This porridge is too hot!\"she exclaimed. So, she tasted the porridge from the second bowl. \"This porridge is too cold,\"she said. So, she tasted the last bowl of porridge. \"Ahhh, this porridge is just right!\"she exclaimed happily. \"What a lovely house this is!\"she said to herself. She looked into the living room. There were chairs, a couch, and a rocking chair. Goldilocks walked over to the rocking chair. It was too big. Then she saw a chair that looked just right. She sat down in it and it was just right. Next, she saw a bed. The first bed she saw was too hard. The next bed was too soft. But the last bed was just right. Goldilocks lay down and went fast asleep. Suddenly, she heard a great noise downstairs. Someone was coming! She jumped out of bed, ran into the bathroom, and hid behind the door. Three bears entered the house. They came into the kitchen. \"Someone has been eating my porridge,\"said the first bear. \"Someone has been eating my porridge,\"said the second bear. \"Someone has been eating my porridge,\"said the third bear. \"And someone has been sitting in my chair.\"\"And someone has been sitting in my chair.\"\"And someone has been sitting in my chair.\"\"And someone has been sleeping in my bed.\"\"And someone has been sleeping in my bed.\"\"And someone has been sleeping in my bed.\"They followed the trail of footprints upstairs. When they got to the bedroom, Goldilocks heard them say, \"Someone has been sleeping in my bed.\"Goldilocks trembled with fear. She opened the bathroom door very slowly and peeked out. She saw three angry bears looking at her. She closed the door quickly and hid behind it. The bears broke down the door, and Goldilocks ran out of the house as fast as she could. She ran so fast that she fell down and broke all the bones in her body. And that is the end of the story of Goldilocks and the Three Bears."; - StringBuilder sb = new StringBuilder(); - pro(v, 0, v.length(), sb, 50); - } - - public static void pro(String text, int n, int length, StringBuilder builder, int ONE_MAX_TEXT){ - if(n < length){ - int start = n; - n += ONE_MAX_TEXT; - if(n > length){ - n = length; - } - String message = text.substring(start, n); - int status = 0; - if(n == length){ - status = 1; - } - message = StringUtils.getSendStr(builder, message, status == 1 ? true : false); - int m = n; - if(StringUtils.isNotEmpty(message)){ - - System.out.println(message); - pro(text, n, length, builder, ONE_MAX_TEXT); - }else if(n < length){ - pro(text, n, length, builder, ONE_MAX_TEXT); - } - } - }*/ - } diff --git a/iot-modules/iot-box-websocket-api/src/main/resources/bootstrap-dev.yml b/iot-modules/iot-box-websocket-api/src/main/resources/bootstrap-dev.yml index 186ee2e..11e8f07 100644 --- a/iot-modules/iot-box-websocket-api/src/main/resources/bootstrap-dev.yml +++ b/iot-modules/iot-box-websocket-api/src/main/resources/bootstrap-dev.yml @@ -51,7 +51,7 @@ tts: #max: # text: 30 #当发送给前端文字超过改值时，分批发送给前端 suanfa: true #nacos控制变化 - url: http://47.110.40.30:18000/run/predict #http://192.168.8.211:18000/run/predict #http://47.110.40.30:18000/run/predict # #算法语音合成 + url: http://192.168.8.211:18000/run/predict # #http://47.110.40.30:18000/run/predict # #算法语音合成 lac: #type: suanfa #nacos控制变化 huburl: http://192.168.8.175:8866/predict/lac