From 2b546f0c056cffb66ad5bb6ca1b1618902b55734 Mon Sep 17 00:00:00 2001 From: wulin Date: Sat, 21 Oct 2023 17:59:43 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E8=87=AA=E5=AE=9A=E4=B9=89?= =?UTF-8?q?=E9=95=BF=E6=96=87=E6=9C=AC=E6=8E=A8=E6=B5=81=E7=9A=84=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 6148 -> 6148 bytes .../com/qiuguo/iot/third/nlp/LacWrapper.java | 4 + .../box/websocket/api/domain/BaseSession.java | 2 +- .../api/handler/BaseWebSocketProcess.java | 199 +++++++++++++----- .../api/handler/BoxWebSocketHandler.java | 1 + .../api/handler/CustomerWebSocketHandler.java | 1 + 6 files changed, 150 insertions(+), 57 deletions(-) create mode 100644 iot-common/iot-third/src/main/java/com/qiuguo/iot/third/nlp/LacWrapper.java diff --git a/.DS_Store b/.DS_Store index 87afb2636d4d28682e8cbfb37a81bd91a391c5fe..c3c62275a9450203642b573c2c6fa01926ae250c 100644 GIT binary patch delta 82 zcmZoMXfc?O%(!DRBh&K91|lqz`58rkJQc=5b&2X~QwtphBQwid9ffK`6AMEf1#>ft l+FDKyQDuGWp!n>Z+`Rm*$-5cl8M`;%X4Gce%+B$b9{|V47wG^1 delta 81 zcmZoMXfc?O%(#6rBh&K91|lqz`58rkJQc=5HSy|d104l(qgow>YD05NBOL`p10& 0) { + //清空 + String msg = old.substring(0, m); + if(msg.replace(" ", "").length() > 0){ + //纯空格的不推送 + sb.setLength(0); + sb.append(old.substring(m)); + return msg; + } + } + sb.append(message); + return null; + } + + + private void sendMoreMsg(BaseSession baseSession, StringBuilder sb, String message, int type){ + message = getSendStr(sb, message); + if(StringUtils.isNotEmpty(message)){ + normalSendMsg(baseSession, message, type, YesNo.NO.getCode()); + } + } + private Mono toQianWen(Action action, BaseSession baseSession, Integer type){ - baseSession.setRequestId(baseSession.getRequestId() + 1); + //baseSession.setRequestId(baseSession.getRequestId() + 1); TongYiCommunicationRest tongYiCommunicationRest = new TongYiCommunicationRest(); tongYiCommunicationRest.setText(action.getAsk()); tongYiCommunicationRest.setStatus("2"); @@ -124,29 +158,8 @@ public class BaseWebSocketProcess { //通知到客户端 MDC.put(Log4Constans.PRINT_LOG_ID, baseSession.getLogId()); if (tongYiCommunicationRest.getRequestId().equals(baseSession.getRequestId())) { - String old = sb.toString() + message; - int d = old.lastIndexOf("，"); - int j = old.lastIndexOf("。"); - int a = old.lastIndexOf("："); - int b = old.lastIndexOf("\n"); - int c = old.lastIndexOf("；"); - int n = old.lastIndexOf("\\n"); - int m = Math.max(Math.max(Math.max(Math.max(d, j), Math.max(a, b)), c), n); - if (m > 0) { - //清空 - String msg = old.substring(0, m); - if(msg.replace(" ", "").length() > 0){ - //纯空格的不推送 - sb.setLength(0); - sb.append(old.substring(m)); - - normalSendMsg(baseSession, msg, type, YesNo.NO.getCode()); - return; - } - - } - sb.append(message); - + //测试后决定是否需要 + sendMoreMsg(baseSession, sb, message, type); return; } log.info("已经有新的请求，不在推送到客户端SN：{} userId:{}", baseSession.getSn(), baseSession.getUserId()); @@ -167,14 +180,7 @@ public class BaseWebSocketProcess { if(data.getCode() == 200){ log.info("千问正常结束"); //保存记录 - DeviceUserTalkRecordEntity talkRecord = new DeviceUserTalkRecordEntity(); - talkRecord.setAskType(AskTypeEnum.TTS.getCode()); - talkRecord.setAskValue(action.getAsk()); - talkRecord.setAskKey(action.getAction()); - talkRecord.setAnswerValue(data.getResut()); - talkRecord.setUserId(baseSession.getUserId()); - talkRecord.setDeviceId(baseSession.getDeviceId()); - return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(b -> { + return saveTalkRecord(baseSession, action, data.getResut()).flatMap(i -> { return Mono.empty(); }); }else{ @@ -185,6 +191,17 @@ public class BaseWebSocketProcess { })/*.subscribeOn(Schedulers.boundedElastic()).subscribe()*/; } + private Mono saveTalkRecord(BaseSession baseSession, Action action, String text){ + DeviceUserTalkRecordEntity talkRecord = new DeviceUserTalkRecordEntity(); + talkRecord.setAskType(AskTypeEnum.TTS.getCode()); + talkRecord.setAskValue(action.getAsk()); + talkRecord.setAskKey(action.getAction()); + talkRecord.setAnswerValue(text); + talkRecord.setUserId(baseSession.getUserId()); + talkRecord.setDeviceId(baseSession.getDeviceId()); + return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord); + } + private Mono toU3DMq(Action action, SystemTalkBindDeviceEntity systemTalkBindDeviceEntity, Long metaId){ U3dMsg u3dMsg = new U3dMsg(); @@ -411,6 +428,7 @@ public class BaseWebSocketProcess { return sendMessage(action, baseSession, action.getSystemTalkAnswerConfigEntity().getAnswerValue(), AskTypeEnum.TTS.getCode()); } }else if(action.getSystemTalkAnswerConfigEntity().getAnswerType().equals(AskTypeEnum.QIU_GUO.getCode())){ + return sendMessage(action, baseSession, action.getSystemTalkAnswerConfigEntity().getAnswerValue(), AskTypeEnum.TTS.getCode()); }else if(action.getSystemTalkAnswerConfigEntity().getAnswerType().equals(AskTypeEnum.TIME.getCode())){ DateTimeFormatter df = DateTimeFormatter.ofPattern(action.getSystemTalkAnswerConfigEntity().getAnswerValue()); @@ -639,13 +657,7 @@ public class BaseWebSocketProcess { } } return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(i ->{ - if(this instanceof BoxWebSocketHandler){ - log.info("果box聊天记录，同步到客户端"); - BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId()); - if(userSession != null){ - sendMsg(userSession, resp); - } - } + sendMsg(baseSession, resp); return Mono.just(true); @@ -687,28 +699,103 @@ public class BaseWebSocketProcess { } private void sendMsg(BaseSession baseSession, BaseMessageResp baseMessageResp) { - if(ttsSuanfa && this instanceof BoxWebSocketHandler){ - String text = baseMessageResp.getText().replace("\n", "").replace("\t", ""); - if(text.startsWith("，") || - text.startsWith("。") || - text.startsWith("：") || - text.startsWith("；") || - text.startsWith("《") || - text.startsWith(" ")){ - //标点符号起始会导致合成的声音第一句话有杂音 - text = text.substring(1); + if(this instanceof BoxWebSocketHandler){ + log.info("果box聊天记录，同步到客户端"); + BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId()); + if(userSession != null){ + sendMsg(userSession, baseMessageResp); + } + if(ttsSuanfa){ + String text = baseMessageResp.getText().replace("\n", "").replace("\t", ""); + if(text.startsWith("，") || + text.startsWith("。") || + text.startsWith("：") || + text.startsWith("；") || + text.startsWith("《") || + text.startsWith(" ")){ + //标点符号起始会导致合成的声音第一句话有杂音 + text = text.substring(1); + } + if(text.length() > ONE_MAX_TEXT){ + StringBuilder builder = new StringBuilder(); + sendAudioMessage(baseSession, + baseMessageResp, + builder, + text, + 0, + text.length() - 1, + baseSession.getRequestId()).subscribe(); + }else{ + BoxMessageResp boxMessageResp = new BoxMessageResp(); + BeanUtils.copyProperties(baseMessageResp, boxMessageResp); + sendAudioMessage(baseSession, boxMessageResp).subscribe(); + } } - audioService.getAudioUrl(text + "。").map(s ->{ - BoxMessageResp boxMessageResp = new BoxMessageResp(); - BeanUtils.copyProperties(baseMessageResp, boxMessageResp); - boxMessageResp.setAudio(s); - sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); - return s; - }).subscribeOn(Schedulers.single()).subscribe(); }else{ sendMsg(baseSession, JSONObject.toJSONString(baseMessageResp)); } + } + /** + * 分批按顺序发送超长ONE_MAX_TEXT文本音频 + * @param baseSession + * @param baseMessageResp + * @param builder + * @param text + * @param n + * @param length + * @return + */ + private Mono sendAudioMessage(BaseSession baseSession, + BaseMessageResp baseMessageResp, + StringBuilder builder, + String text, + int n, + int length, + Long requestId){ + if(n < length && baseSession.getRequestId().equals(requestId)){ + n += ONE_MAX_TEXT; + if(n > length){ + n = length; + } + String message = text.substring(n - ONE_MAX_TEXT, n); + int status = 0; + if(n == length){ + message += "。"; + status = 1; + builder.setLength(0); + } + message = getSendStr(builder, message); + int m = n; + if(StringUtils.isNotEmpty(message)){ + BoxMessageResp boxMessageResp = new BoxMessageResp(); + BeanUtils.copyProperties(baseMessageResp, boxMessageResp); + boxMessageResp.setText(message); + boxMessageResp.getTts().setStatus(status); + return sendAudioMessage(baseSession, boxMessageResp).flatMap(s -> { + return sendAudioMessage(baseSession, baseMessageResp, builder, text, m, length, requestId); + }); + }else if(n < length){ + return sendAudioMessage(baseSession, baseMessageResp, builder, text, m, length, requestId); + } + } + + return Mono.just(""); + } + + /** + * 一次性发送文本 + * @param baseSession + * @param boxMessageResp + * @return + */ + private Mono sendAudioMessage(BaseSession baseSession, BoxMessageResp boxMessageResp){ + return audioService.getAudioUrl(boxMessageResp.getText() + "。").map(s ->{ + + boxMessageResp.setAudio(s); + sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); + return s; + });//.subscribeOn(Schedulers.single()).subscribe(); } private void sendMsg(BaseSession baseSession, String msg) { diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java index e863a62..37f913d 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java @@ -133,6 +133,7 @@ public class BoxWebSocketHandler extends BaseWebSocketProcess implements WebSock } log.info("收到SN:{}，消息：{}", boxTalkMessage.getSn(), boxTalkMessage.getMessage()); return nlpService.getActionWithLacSingle(boxSession.getUserId(), boxTalkMessage.getMessage()).defaultIfEmpty(new Actions()).flatMap(actions -> { + boxSession.setRequestId(boxSession.getRequestId() + 1); return processAction(actions, boxSession); //return Mono.empty(); }); diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java index ce49cbf..27fed7c 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java @@ -121,6 +121,7 @@ public class CustomerWebSocketHandler extends BaseWebSocketProcess implements We return nlpService.getActionWithLacSingle(userSession.getUserId(), userTalkMessage.getMessage()) .defaultIfEmpty(new Actions()).flatMap(actions -> { //处理 + userSession.setRequestId(userSession.getRequestId() + 1); return processAction(actions, userSession); });