diff --git a/.DS_Store b/.DS_Store index 87afb26..c3c6227 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/nlp/LacWrapper.java b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/nlp/LacWrapper.java new file mode 100644 index 0000000..8be0514 --- /dev/null +++ b/iot-common/iot-third/src/main/java/com/qiuguo/iot/third/nlp/LacWrapper.java @@ -0,0 +1,4 @@ +package com.qiuguo.iot.third.nlp; + +public class LacWrapper { +} diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/domain/BaseSession.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/domain/BaseSession.java index e761e2c..62e5183 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/domain/BaseSession.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/domain/BaseSession.java @@ -45,7 +45,7 @@ public class BaseSession { protected MusicResp music; /** - * 调用千问请求序号，每次确定调用前++ + * 问题Id，每次++ */ protected Long requestId = 0L; } diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BaseWebSocketProcess.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BaseWebSocketProcess.java index 63e1452..25668b4 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BaseWebSocketProcess.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BaseWebSocketProcess.java @@ -47,6 +47,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.web.reactive.socket.WebSocketSession; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Scheduler; import reactor.core.scheduler.Schedulers; import javax.annotation.Resource; @@ -93,6 +94,8 @@ public class BaseWebSocketProcess { @Value("${tts.suanfa}") boolean ttsSuanfa; + private static int ONE_MAX_TEXT = 30; + @Autowired protected SystemTalkBindU3dService systemTalkBindU3dService; @@ -104,10 +107,41 @@ public class BaseWebSocketProcess { protected static String apiType = "api-type"; protected static String apiToken = "api-token"; + private String getSendStr(StringBuilder sb, String message){ + String old = sb.toString() + message; + int d = old.lastIndexOf("，"); + int j = old.lastIndexOf("。"); + int a = old.lastIndexOf("："); + int b = old.lastIndexOf("\n"); + int c = old.lastIndexOf("；"); + int n = old.lastIndexOf("\\n"); + int m = Math.max(Math.max(Math.max(Math.max(d, j), Math.max(a, b)), c), n); + if (m > 0) { + //清空 + String msg = old.substring(0, m); + if(msg.replace(" ", "").length() > 0){ + //纯空格的不推送 + sb.setLength(0); + sb.append(old.substring(m)); + return msg; + } + } + sb.append(message); + return null; + } + + + private void sendMoreMsg(BaseSession baseSession, StringBuilder sb, String message, int type){ + message = getSendStr(sb, message); + if(StringUtils.isNotEmpty(message)){ + normalSendMsg(baseSession, message, type, YesNo.NO.getCode()); + } + } + private Mono toQianWen(Action action, BaseSession baseSession, Integer type){ - baseSession.setRequestId(baseSession.getRequestId() + 1); + //baseSession.setRequestId(baseSession.getRequestId() + 1); TongYiCommunicationRest tongYiCommunicationRest = new TongYiCommunicationRest(); tongYiCommunicationRest.setText(action.getAsk()); tongYiCommunicationRest.setStatus("2"); @@ -124,29 +158,8 @@ public class BaseWebSocketProcess { //通知到客户端 MDC.put(Log4Constans.PRINT_LOG_ID, baseSession.getLogId()); if (tongYiCommunicationRest.getRequestId().equals(baseSession.getRequestId())) { - String old = sb.toString() + message; - int d = old.lastIndexOf("，"); - int j = old.lastIndexOf("。"); - int a = old.lastIndexOf("："); - int b = old.lastIndexOf("\n"); - int c = old.lastIndexOf("；"); - int n = old.lastIndexOf("\\n"); - int m = Math.max(Math.max(Math.max(Math.max(d, j), Math.max(a, b)), c), n); - if (m > 0) { - //清空 - String msg = old.substring(0, m); - if(msg.replace(" ", "").length() > 0){ - //纯空格的不推送 - sb.setLength(0); - sb.append(old.substring(m)); - - normalSendMsg(baseSession, msg, type, YesNo.NO.getCode()); - return; - } - - } - sb.append(message); - + //测试后决定是否需要 + sendMoreMsg(baseSession, sb, message, type); return; } log.info("已经有新的请求，不在推送到客户端SN：{} userId:{}", baseSession.getSn(), baseSession.getUserId()); @@ -167,14 +180,7 @@ public class BaseWebSocketProcess { if(data.getCode() == 200){ log.info("千问正常结束"); //保存记录 - DeviceUserTalkRecordEntity talkRecord = new DeviceUserTalkRecordEntity(); - talkRecord.setAskType(AskTypeEnum.TTS.getCode()); - talkRecord.setAskValue(action.getAsk()); - talkRecord.setAskKey(action.getAction()); - talkRecord.setAnswerValue(data.getResut()); - talkRecord.setUserId(baseSession.getUserId()); - talkRecord.setDeviceId(baseSession.getDeviceId()); - return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(b -> { + return saveTalkRecord(baseSession, action, data.getResut()).flatMap(i -> { return Mono.empty(); }); }else{ @@ -185,6 +191,17 @@ public class BaseWebSocketProcess { })/*.subscribeOn(Schedulers.boundedElastic()).subscribe()*/; } + private Mono saveTalkRecord(BaseSession baseSession, Action action, String text){ + DeviceUserTalkRecordEntity talkRecord = new DeviceUserTalkRecordEntity(); + talkRecord.setAskType(AskTypeEnum.TTS.getCode()); + talkRecord.setAskValue(action.getAsk()); + talkRecord.setAskKey(action.getAction()); + talkRecord.setAnswerValue(text); + talkRecord.setUserId(baseSession.getUserId()); + talkRecord.setDeviceId(baseSession.getDeviceId()); + return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord); + } + private Mono toU3DMq(Action action, SystemTalkBindDeviceEntity systemTalkBindDeviceEntity, Long metaId){ U3dMsg u3dMsg = new U3dMsg(); @@ -411,6 +428,7 @@ public class BaseWebSocketProcess { return sendMessage(action, baseSession, action.getSystemTalkAnswerConfigEntity().getAnswerValue(), AskTypeEnum.TTS.getCode()); } }else if(action.getSystemTalkAnswerConfigEntity().getAnswerType().equals(AskTypeEnum.QIU_GUO.getCode())){ + return sendMessage(action, baseSession, action.getSystemTalkAnswerConfigEntity().getAnswerValue(), AskTypeEnum.TTS.getCode()); }else if(action.getSystemTalkAnswerConfigEntity().getAnswerType().equals(AskTypeEnum.TIME.getCode())){ DateTimeFormatter df = DateTimeFormatter.ofPattern(action.getSystemTalkAnswerConfigEntity().getAnswerValue()); @@ -639,13 +657,7 @@ public class BaseWebSocketProcess { } } return deviceUserTalkRecordService.insertDeviceUserTalkRecord(talkRecord).flatMap(i ->{ - if(this instanceof BoxWebSocketHandler){ - log.info("果box聊天记录，同步到客户端"); - BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId()); - if(userSession != null){ - sendMsg(userSession, resp); - } - } + sendMsg(baseSession, resp); return Mono.just(true); @@ -687,28 +699,103 @@ public class BaseWebSocketProcess { } private void sendMsg(BaseSession baseSession, BaseMessageResp baseMessageResp) { - if(ttsSuanfa && this instanceof BoxWebSocketHandler){ - String text = baseMessageResp.getText().replace("\n", "").replace("\t", ""); - if(text.startsWith("，") || - text.startsWith("。") || - text.startsWith("：") || - text.startsWith("；") || - text.startsWith("《") || - text.startsWith(" ")){ - //标点符号起始会导致合成的声音第一句话有杂音 - text = text.substring(1); + if(this instanceof BoxWebSocketHandler){ + log.info("果box聊天记录，同步到客户端"); + BaseSession userSession = getUserSessionWithUserId(baseSession.getUserId()); + if(userSession != null){ + sendMsg(userSession, baseMessageResp); + } + if(ttsSuanfa){ + String text = baseMessageResp.getText().replace("\n", "").replace("\t", ""); + if(text.startsWith("，") || + text.startsWith("。") || + text.startsWith("：") || + text.startsWith("；") || + text.startsWith("《") || + text.startsWith(" ")){ + //标点符号起始会导致合成的声音第一句话有杂音 + text = text.substring(1); + } + if(text.length() > ONE_MAX_TEXT){ + StringBuilder builder = new StringBuilder(); + sendAudioMessage(baseSession, + baseMessageResp, + builder, + text, + 0, + text.length() - 1, + baseSession.getRequestId()).subscribe(); + }else{ + BoxMessageResp boxMessageResp = new BoxMessageResp(); + BeanUtils.copyProperties(baseMessageResp, boxMessageResp); + sendAudioMessage(baseSession, boxMessageResp).subscribe(); + } } - audioService.getAudioUrl(text + "。").map(s ->{ - BoxMessageResp boxMessageResp = new BoxMessageResp(); - BeanUtils.copyProperties(baseMessageResp, boxMessageResp); - boxMessageResp.setAudio(s); - sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); - return s; - }).subscribeOn(Schedulers.single()).subscribe(); }else{ sendMsg(baseSession, JSONObject.toJSONString(baseMessageResp)); } + } + /** + * 分批按顺序发送超长ONE_MAX_TEXT文本音频 + * @param baseSession + * @param baseMessageResp + * @param builder + * @param text + * @param n + * @param length + * @return + */ + private Mono sendAudioMessage(BaseSession baseSession, + BaseMessageResp baseMessageResp, + StringBuilder builder, + String text, + int n, + int length, + Long requestId){ + if(n < length && baseSession.getRequestId().equals(requestId)){ + n += ONE_MAX_TEXT; + if(n > length){ + n = length; + } + String message = text.substring(n - ONE_MAX_TEXT, n); + int status = 0; + if(n == length){ + message += "。"; + status = 1; + builder.setLength(0); + } + message = getSendStr(builder, message); + int m = n; + if(StringUtils.isNotEmpty(message)){ + BoxMessageResp boxMessageResp = new BoxMessageResp(); + BeanUtils.copyProperties(baseMessageResp, boxMessageResp); + boxMessageResp.setText(message); + boxMessageResp.getTts().setStatus(status); + return sendAudioMessage(baseSession, boxMessageResp).flatMap(s -> { + return sendAudioMessage(baseSession, baseMessageResp, builder, text, m, length, requestId); + }); + }else if(n < length){ + return sendAudioMessage(baseSession, baseMessageResp, builder, text, m, length, requestId); + } + } + + return Mono.just(""); + } + + /** + * 一次性发送文本 + * @param baseSession + * @param boxMessageResp + * @return + */ + private Mono sendAudioMessage(BaseSession baseSession, BoxMessageResp boxMessageResp){ + return audioService.getAudioUrl(boxMessageResp.getText() + "。").map(s ->{ + + boxMessageResp.setAudio(s); + sendMsg(baseSession, JSONObject.toJSONString(boxMessageResp)); + return s; + });//.subscribeOn(Schedulers.single()).subscribe(); } private void sendMsg(BaseSession baseSession, String msg) { diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java index e863a62..37f913d 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/BoxWebSocketHandler.java @@ -133,6 +133,7 @@ public class BoxWebSocketHandler extends BaseWebSocketProcess implements WebSock } log.info("收到SN:{}，消息：{}", boxTalkMessage.getSn(), boxTalkMessage.getMessage()); return nlpService.getActionWithLacSingle(boxSession.getUserId(), boxTalkMessage.getMessage()).defaultIfEmpty(new Actions()).flatMap(actions -> { + boxSession.setRequestId(boxSession.getRequestId() + 1); return processAction(actions, boxSession); //return Mono.empty(); }); diff --git a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java index ce49cbf..27fed7c 100644 --- a/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java +++ b/iot-modules/iot-box-websocket-api/src/main/java/com/qiuguo/iot/box/websocket/api/handler/CustomerWebSocketHandler.java @@ -121,6 +121,7 @@ public class CustomerWebSocketHandler extends BaseWebSocketProcess implements We return nlpService.getActionWithLacSingle(userSession.getUserId(), userTalkMessage.getMessage()) .defaultIfEmpty(new Actions()).flatMap(actions -> { //处理 + userSession.setRequestId(userSession.getRequestId() + 1); return processAction(actions, userSession); });