diff --git a/flx-ai/pom.xml b/flx-ai/pom.xml index 6643cdd..49ba9e5 100644 --- a/flx-ai/pom.xml +++ b/flx-ai/pom.xml @@ -148,6 +148,12 @@ knife4j-spring-boot-starter ${knife4j.version} + + + com.hankcs + hanlp + portable-1.8.6 + diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/controller/ChunksController.java b/flx-ai/src/main/java/com/pjilisense/flxai/controller/ChunksController.java new file mode 100644 index 0000000..23531a0 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/controller/ChunksController.java @@ -0,0 +1,93 @@ +package com.pjilisense.flxai.controller; + +import com.pjilisense.flxai.base.constant.Constant; +import com.pjilisense.flxai.dto.FilesDTO; +import com.pjilisense.flxai.page.PageData; +import com.pjilisense.flxai.utils.Result; +import com.pjilisense.flxai.dto.ChunksDTO; +import com.pjilisense.flxai.service.ChunksService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@RestController +@RequestMapping(Constant.BASE_PATH+"chunks") +@Api(tags="${comments}") +public class ChunksController { + + private static Logger logger = LoggerFactory.getLogger(ChunksController.class); + + @Autowired + private ChunksService chunksService; + + @GetMapping("page") + @ApiOperation("分页") + @ApiImplicitParams({ + @ApiImplicitParam(name = Constant.PAGE, value = "当前页码,从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) , + @ApiImplicitParam(name = Constant.ORDER, value = "排序方式,可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class) + }) + public Result> page(@ApiIgnore @RequestParam Map params){ + PageData page = chunksService.page(params); + + return new Result>().ok(page); + } + + @GetMapping("{id}") + @ApiOperation("信息") + public Result get(@PathVariable("id") Long id){ + ChunksDTO data = chunksService.get(id); + + return new Result().ok(data); + } + @PostMapping("/search") + @ApiOperation("模糊检索") + public Result searchByText(@RequestBody ChunksDTO dto){ + List> list =chunksService.search(dto.getUserId(),dto.getText()); + return new Result().ok(list); + } + @PostMapping + @ApiOperation("保存") + public Result save(@RequestBody ChunksDTO dto){ + chunksService.save(dto); + + return new Result(); + } + + @PutMapping + @ApiOperation("修改") + public Result update(@RequestBody ChunksDTO dto){ + + chunksService.update(dto); + + return new Result(); + } + + @DeleteMapping + @ApiOperation("删除") + public Result delete(@RequestBody Long[] ids){ + //效验数据 + chunksService.delete(ids); + + return new Result(); + } + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/controller/FileChunksController.java b/flx-ai/src/main/java/com/pjilisense/flxai/controller/FileChunksController.java new file mode 100644 index 0000000..ca43b0e --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/controller/FileChunksController.java @@ -0,0 +1,88 @@ +package com.pjilisense.flxai.controller; + +import com.pjilisense.flxai.base.constant.Constant; +import com.pjilisense.flxai.page.PageData; +import com.pjilisense.flxai.utils.Result; +import com.pjilisense.flxai.dto.FileChunksDTO; +import com.pjilisense.flxai.service.FileChunksService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@RestController +@RequestMapping(Constant.BASE_PATH+"filechunks") +@Api(tags="${comments}") +public class FileChunksController { + + private static Logger logger = LoggerFactory.getLogger(FileChunksController.class); + + @Autowired + private FileChunksService fileChunksService; + + @GetMapping("page") + @ApiOperation("分页") + @ApiImplicitParams({ + @ApiImplicitParam(name = Constant.PAGE, value = "当前页码,从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) , + @ApiImplicitParam(name = Constant.ORDER, value = "排序方式,可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class) + }) + public Result> page(@ApiIgnore @RequestParam Map params){ + PageData page = fileChunksService.page(params); + + return new Result>().ok(page); + } + + @GetMapping("{id}") + @ApiOperation("信息") + public Result get(@PathVariable("id") Long id){ + FileChunksDTO data = fileChunksService.get(id); + + return new Result().ok(data); + } + + @PostMapping + @ApiOperation("保存") + public Result save(@RequestBody FileChunksDTO dto){ + + fileChunksService.save(dto); + + return new Result(); + } + + @PutMapping + @ApiOperation("修改") + public Result update(@RequestBody FileChunksDTO dto){ + + fileChunksService.update(dto); + + return new Result(); + } + + @DeleteMapping + @ApiOperation("删除") + public Result delete(@RequestBody Long[] ids){ + + fileChunksService.delete(ids); + + return new Result(); + } + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/controller/GlobalFilesController.java b/flx-ai/src/main/java/com/pjilisense/flxai/controller/GlobalFilesController.java new file mode 100644 index 0000000..cea635d --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/controller/GlobalFilesController.java @@ -0,0 +1,88 @@ +package com.pjilisense.flxai.controller; + +import com.pjilisense.flxai.base.constant.Constant; +import com.pjilisense.flxai.page.PageData; +import com.pjilisense.flxai.utils.Result; +import com.pjilisense.flxai.dto.GlobalFilesDTO; +import com.pjilisense.flxai.service.GlobalFilesService; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiImplicitParam; +import io.swagger.annotations.ApiImplicitParams; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.*; +import springfox.documentation.annotations.ApiIgnore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@RestController +@RequestMapping(Constant.BASE_PATH+"globalfiles") +@Api(tags="${comments}") +public class GlobalFilesController { + + private static Logger logger = LoggerFactory.getLogger(GlobalFilesController.class); + + @Autowired + private GlobalFilesService globalFilesService; + + @GetMapping("page") + @ApiOperation("分页") + @ApiImplicitParams({ + @ApiImplicitParam(name = Constant.PAGE, value = "当前页码,从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) , + @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) , + @ApiImplicitParam(name = Constant.ORDER, value = "排序方式,可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class) + }) + public Result> page(@ApiIgnore @RequestParam Map params){ + PageData page = globalFilesService.page(params); + + return new Result>().ok(page); + } + + @GetMapping("{id}") + @ApiOperation("信息") + public Result get(@PathVariable("id") Long id){ + GlobalFilesDTO data = globalFilesService.get(id); + + return new Result().ok(data); + } + + @PostMapping + @ApiOperation("保存") + public Result save(@RequestBody GlobalFilesDTO dto){ + + globalFilesService.save(dto); + + return new Result(); + } + + @PutMapping + @ApiOperation("修改") + public Result update(@RequestBody GlobalFilesDTO dto){ + + globalFilesService.update(dto); + + return new Result(); + } + + @DeleteMapping + @ApiOperation("删除") + public Result delete(@RequestBody Long[] ids){ + + globalFilesService.delete(ids); + + return new Result(); + } + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dao/ChunksDao.java b/flx-ai/src/main/java/com/pjilisense/flxai/dao/ChunksDao.java new file mode 100644 index 0000000..f4debc9 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dao/ChunksDao.java @@ -0,0 +1,20 @@ +package com.pjilisense.flxai.dao; + +import com.pjilisense.flxai.base.dao.BaseDao; +import com.pjilisense.flxai.entity.ChunksEntity; +import org.apache.ibatis.annotations.Mapper; + +import java.util.List; +import java.util.Map; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Mapper +public interface ChunksDao extends BaseDao { + List> queryCsv(Map map); + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dao/FileChunksDao.java b/flx-ai/src/main/java/com/pjilisense/flxai/dao/FileChunksDao.java new file mode 100644 index 0000000..0e620ff --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dao/FileChunksDao.java @@ -0,0 +1,16 @@ +package com.pjilisense.flxai.dao; + +import com.pjilisense.flxai.base.dao.BaseDao; +import com.pjilisense.flxai.entity.FileChunksEntity; +import org.apache.ibatis.annotations.Mapper; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Mapper +public interface FileChunksDao extends BaseDao { + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dao/GlobalFilesDao.java b/flx-ai/src/main/java/com/pjilisense/flxai/dao/GlobalFilesDao.java new file mode 100644 index 0000000..85e132e --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dao/GlobalFilesDao.java @@ -0,0 +1,16 @@ +package com.pjilisense.flxai.dao; + +import com.pjilisense.flxai.base.dao.BaseDao; +import com.pjilisense.flxai.entity.GlobalFilesEntity; +import org.apache.ibatis.annotations.Mapper; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Mapper +public interface GlobalFilesDao extends BaseDao { + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dto/ChunksDTO.java b/flx-ai/src/main/java/com/pjilisense/flxai/dto/ChunksDTO.java new file mode 100644 index 0000000..0e17396 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dto/ChunksDTO.java @@ -0,0 +1,64 @@ +package com.pjilisense.flxai.dto; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.Map; +import java.util.UUID; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@ApiModel(value = "${comments}") +public class ChunksDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "$column.comments") + private UUID id; + + @ApiModelProperty(value = "$column.comments") + private String text; + + @ApiModelProperty(value = "$column.comments") + private String abstractx; + + @ApiModelProperty(value = "$column.comments") + private Map metadata; + + @ApiModelProperty(value = "$column.comments") + private Integer index; + + @ApiModelProperty(value = "$column.comments") + private String type; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime createdAt; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime updatedAt; + + @ApiModelProperty(value = "$column.comments") + private String userId; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime accessedAt; + + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dto/FileChunksDTO.java b/flx-ai/src/main/java/com/pjilisense/flxai/dto/FileChunksDTO.java new file mode 100644 index 0000000..722902d --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dto/FileChunksDTO.java @@ -0,0 +1,38 @@ +package com.pjilisense.flxai.dto; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.UUID; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@ApiModel(value = "${comments}") +public class FileChunksDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "$column.comments") + private String fileId; + + @ApiModelProperty(value = "$column.comments") + private UUID chunkId; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime createdAt; + + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/dto/GlobalFilesDTO.java b/flx-ai/src/main/java/com/pjilisense/flxai/dto/GlobalFilesDTO.java new file mode 100644 index 0000000..024ad5e --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/dto/GlobalFilesDTO.java @@ -0,0 +1,52 @@ +package com.pjilisense.flxai.dto; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.io.Serializable; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.Map; + + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@ApiModel(value = "${comments}") +public class GlobalFilesDTO implements Serializable { + private static final long serialVersionUID = 1L; + + @ApiModelProperty(value = "$column.comments") + private String hashId; + + @ApiModelProperty(value = "$column.comments") + private String fileType; + + @ApiModelProperty(value = "$column.comments") + private Integer size; + + @ApiModelProperty(value = "$column.comments") + private String url; + + @ApiModelProperty(value = "$column.comments") + private Map metadata; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime createdAt; + + @ApiModelProperty(value = "$column.comments") + @JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class) + @JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class) + private OffsetDateTime accessedAt; + + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/entity/ChunksEntity.java b/flx-ai/src/main/java/com/pjilisense/flxai/entity/ChunksEntity.java new file mode 100644 index 0000000..7cb9ff0 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/entity/ChunksEntity.java @@ -0,0 +1,67 @@ +package com.pjilisense.flxai.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import com.pjilisense.flxai.handle.JsonbTypeHandler; +import com.pjilisense.flxai.handle.UUIDTypeHandler; +import lombok.Data; + +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.Map; +import java.util.UUID; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@TableName("chunks") +public class ChunksEntity { + + /** + * $column.comments + */ + @TableField(typeHandler = UUIDTypeHandler.class) + private UUID id; + /** + * $column.comments + */ + private String text; + /** + * $column.comments + */ + @TableField(value="abstract") + private String abstractx; + /** + * $column.comments + */ + @TableField(value = "metadata", typeHandler = JsonbTypeHandler.class) + private Map metadata; + /** + * $column.comments + */ + private Integer index; + /** + * $column.comments + */ + private String type; + /** + * $column.comments + */ + private OffsetDateTime createdAt; + /** + * $column.comments + */ + private OffsetDateTime updatedAt; + /** + * $column.comments + */ + private String userId; + /** + * $column.comments + */ + private OffsetDateTime accessedAt; +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/entity/FileChunksEntity.java b/flx-ai/src/main/java/com/pjilisense/flxai/entity/FileChunksEntity.java new file mode 100644 index 0000000..c095c87 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/entity/FileChunksEntity.java @@ -0,0 +1,35 @@ +package com.pjilisense.flxai.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import com.pjilisense.flxai.handle.UUIDTypeHandler; +import lombok.Data; + +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.UUID; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@TableName("file_chunks") +public class FileChunksEntity { + + /** + * $column.comments + */ + private String fileId; + /** + * $column.comments + */ + @TableField(typeHandler = UUIDTypeHandler.class) + private UUID chunkId; + /** + * $column.comments + */ + private OffsetDateTime createdAt; +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/entity/GlobalFilesEntity.java b/flx-ai/src/main/java/com/pjilisense/flxai/entity/GlobalFilesEntity.java new file mode 100644 index 0000000..246931e --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/entity/GlobalFilesEntity.java @@ -0,0 +1,51 @@ +package com.pjilisense.flxai.entity; + +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableName; +import com.pjilisense.flxai.handle.JsonbTypeHandler; +import lombok.Data; + +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.Map; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Data +@TableName("global_files") +public class GlobalFilesEntity { + + /** + * $column.comments + */ + private String hashId; + /** + * $column.comments + */ + private String fileType; + /** + * $column.comments + */ + private Integer size; + /** + * $column.comments + */ + private String url; + /** + * $column.comments + */ + @TableField(value = "metadata", typeHandler = JsonbTypeHandler.class) + private Map metadata; + /** + * $column.comments + */ + private OffsetDateTime createdAt; + /** + * $column.comments + */ + private OffsetDateTime accessedAt; +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/ChunksService.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/ChunksService.java new file mode 100644 index 0000000..e749899 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/ChunksService.java @@ -0,0 +1,19 @@ +package com.pjilisense.flxai.service; + +import com.pjilisense.flxai.base.service.CrudService; +import com.pjilisense.flxai.dto.ChunksDTO; +import com.pjilisense.flxai.entity.ChunksEntity; + +import java.util.List; +import java.util.Map; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +public interface ChunksService extends CrudService { + + List> search(String userid,String text); +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/FileChunksService.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/FileChunksService.java new file mode 100644 index 0000000..8abd7df --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/FileChunksService.java @@ -0,0 +1,15 @@ +package com.pjilisense.flxai.service; + +import com.pjilisense.flxai.base.service.CrudService; +import com.pjilisense.flxai.dto.FileChunksDTO; +import com.pjilisense.flxai.entity.FileChunksEntity; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +public interface FileChunksService extends CrudService { + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/GlobalFilesService.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/GlobalFilesService.java new file mode 100644 index 0000000..d94777a --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/GlobalFilesService.java @@ -0,0 +1,15 @@ +package com.pjilisense.flxai.service; + +import com.pjilisense.flxai.base.service.CrudService; +import com.pjilisense.flxai.dto.GlobalFilesDTO; +import com.pjilisense.flxai.entity.GlobalFilesEntity; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +public interface GlobalFilesService extends CrudService { + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/ChunksServiceImpl.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/ChunksServiceImpl.java new file mode 100644 index 0000000..7cf44ac --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/ChunksServiceImpl.java @@ -0,0 +1,244 @@ +package com.pjilisense.flxai.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.hankcs.hanlp.HanLP; +import com.hankcs.hanlp.seg.Segment; +import com.hankcs.hanlp.seg.common.Term; +import com.pjilisense.flxai.base.service.impl.CrudServiceImpl; +import com.pjilisense.flxai.dao.ChunksDao; +import com.pjilisense.flxai.dto.ChunksDTO; +import com.pjilisense.flxai.entity.ChunksEntity; +import com.pjilisense.flxai.service.ChunksService; +import cn.hutool.core.util.StrUtil; +import org.springframework.stereotype.Service; + +import java.math.BigDecimal; +import java.util.*; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Service +public class ChunksServiceImpl extends CrudServiceImpl implements ChunksService { + + @Override + public QueryWrapper getWrapper(Map params){ + String id = (String)params.get("id"); + + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq(StrUtil.isNotBlank(id), "id", id); + + return wrapper; + } + + + @Override + public List> search(String userid,String keyText) { + Segment seg= HanLP.newSegment(); + seg.enableCustomDictionary(false); +// String keyText="我爱北京天安门"; +// keyText="HanLP是由一系列NLP工具组成的开源项目,其中包含了中文分词、词性标注、命名实体识别等功能。"; + List lst = seg.seg(keyText); + ArrayList listWord=processTermList(lst); + //ArrayList listSentence=processTermListx(lst); + LinkedHashSet set = new LinkedHashSet<>(); + set.addAll(listWord); + //set.addAll(listSentence); + listWord.clear(); + //listSentence.clear(); + ArrayList wordList=processTermList(lst); + wordList.addAll(set); + set.clear(); + HashMap map = new HashMap<>(); + map.put("userid",userid); + map.put("wordList",wordList); + List> list = baseDao.queryCsv(map); + List> listRet = new ArrayList<>() ; + if(list!=null && list.size()>0) { + ArrayList listText=processTermListN0Com(lst); + List idList = new ArrayList<>(); + for (Map mp: list){ + String ttext = String.valueOf(mp.get("text")); + int pos = ttext.indexOf(','); + String question = ttext.substring(0,pos); + String answer = ttext.substring(pos+1); + mp.put("question",question); + mp.put("answer",answer); + List lst2 = seg.seg(question); + ArrayList listQuestion=processTermListN0Com(lst2); + double similarity =calculateSimilarity(listText, listQuestion); + BigDecimal bd = new BigDecimal(similarity); + if(bd.compareTo(BigDecimal.ZERO.stripTrailingZeros())==0){ + idList.add(String.valueOf(mp.get("id"))); + } else { + System.out.println(bd.toString()); + mp.put("similarity",bd.toString()); + } + } + for(Map mp: list){ + String id=String.valueOf(mp.get("id")); + if(!idList.contains(id)){ + listRet.add(mp); + } + } + list.clear(); + list.addAll(listRet); + Collections.sort(list, new Comparator>() { + @Override + public int compare(Map o1, Map o2) { + BigDecimal bd1 = new BigDecimal(String.valueOf(o1.get("similarity"))); + BigDecimal bd2 = new BigDecimal(String.valueOf(o2.get("similarity"))); + return bd1.compareTo(bd2)*(-1); + } + }); + } + return list; + } + + + private static ArrayList processTermList(List lst) { + ArrayList ret = new ArrayList<>(); + StringBuilder sd = new StringBuilder(); + for(int i=0;i0) { + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + continue; + }else { + String word = term.word; + if(word.length()==1) { + sd.append(word); + if(i+1< lst.size()) { + Term termx = lst.get(i+1); + if(!termx.nature.startsWith('w')) { + String wordx = termx.word; + sd.append(wordx); + if(wordx.length()>1) { + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + i++; + } + } + } else { + sd.append(word); + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + } + + } +// System.out.println("ret="); +// for(String term : ret) { +// System.out.println(term); +// } + return ret; + } + + private static ArrayList processTermListN0Com(List lst) { + ArrayList ret = new ArrayList<>(); + StringBuilder sd = new StringBuilder(); + for(int i=0;i0) { + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + continue; + }else { + String word = term.word; + if(word.length()==1) { + sd.append(word); + if(i+1< lst.size()) { + Term termx = lst.get(i+1); + if(!termx.nature.startsWith('w')) { + String wordx = termx.word; + sd.append(wordx); + if(wordx.length()>0) { + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + i++; + } + } + } else { + sd.append(word); + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + } + + } +// System.out.println("ret="); +// for(String term : ret) { +// System.out.println(term); +// } + return ret; + } + private static ArrayList processTermListx(List lst) { + ArrayList ret = new ArrayList<>(); + StringBuilder sd = new StringBuilder(); + for(int i=0;i0) { + ret.add(sd.toString()); + sd = sd.delete(0,sd.length()); + } + continue; + }else { + String word = term.word; + sd.append(word); + } + } +// System.out.println("ret="); +// for(String term : ret) { +// System.out.println(term); +// } + return ret; + } + + // 创建单词频率映射 + private Map createFrequencyMap(List words) { + Map freqMap = new HashMap<>(); + for (String word : words) { + freqMap.put(word, freqMap.getOrDefault(word, 0) + 1); + } + return freqMap; + } + + // 计算余弦相似度 + private double calculateSimilarity(List words1, List words2) { +// List words1 = preprocessText(text1); +// List words2 = preprocessText(text2); + + Map freqMap1 = createFrequencyMap(words1); + Map freqMap2 = createFrequencyMap(words2); + + double dotProduct = 0.0; + double magnitude1 = 0.0; + double magnitude2 = 0.0; + + for (String word : freqMap1.keySet()) { + int freq1 = freqMap1.get(word); + magnitude1 += freq1 * freq1; + if (freqMap2.containsKey(word)) { + int freq2 = freqMap2.get(word); + dotProduct += freq1 * freq2; + } + } + + for (int freq : freqMap2.values()) { + magnitude2 += freq * freq; + } + + return dotProduct / (Math.sqrt(magnitude1) * Math.sqrt(magnitude2)); + } +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/FileChunksServiceImpl.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/FileChunksServiceImpl.java new file mode 100644 index 0000000..01e1fa4 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/FileChunksServiceImpl.java @@ -0,0 +1,34 @@ +package com.pjilisense.flxai.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.pjilisense.flxai.base.service.impl.CrudServiceImpl; +import com.pjilisense.flxai.dao.FileChunksDao; +import com.pjilisense.flxai.dto.FileChunksDTO; +import com.pjilisense.flxai.entity.FileChunksEntity; +import com.pjilisense.flxai.service.FileChunksService; +import cn.hutool.core.util.StrUtil; +import org.springframework.stereotype.Service; + +import java.util.Map; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Service +public class FileChunksServiceImpl extends CrudServiceImpl implements FileChunksService { + + @Override + public QueryWrapper getWrapper(Map params){ + String id = (String)params.get("id"); + + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq(StrUtil.isNotBlank(id), "id", id); + + return wrapper; + } + + +} \ No newline at end of file diff --git a/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/GlobalFilesServiceImpl.java b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/GlobalFilesServiceImpl.java new file mode 100644 index 0000000..1b212b7 --- /dev/null +++ b/flx-ai/src/main/java/com/pjilisense/flxai/service/impl/GlobalFilesServiceImpl.java @@ -0,0 +1,34 @@ +package com.pjilisense.flxai.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.pjilisense.flxai.base.service.impl.CrudServiceImpl; +import com.pjilisense.flxai.dao.GlobalFilesDao; +import com.pjilisense.flxai.dto.GlobalFilesDTO; +import com.pjilisense.flxai.entity.GlobalFilesEntity; +import com.pjilisense.flxai.service.GlobalFilesService; +import cn.hutool.core.util.StrUtil; +import org.springframework.stereotype.Service; + +import java.util.Map; + +/** + * ${comments} + * + * @author liushujing liushujing@philisense.com + * @since 1.0.0 2025-02-10 + */ +@Service +public class GlobalFilesServiceImpl extends CrudServiceImpl implements GlobalFilesService { + + @Override + public QueryWrapper getWrapper(Map params){ + String id = (String)params.get("id"); + + QueryWrapper wrapper = new QueryWrapper<>(); + wrapper.eq(StrUtil.isNotBlank(id), "id", id); + + return wrapper; + } + + +} \ No newline at end of file diff --git a/flx-ai/src/main/resources/mapper/postgres/ChunksDao.xml b/flx-ai/src/main/resources/mapper/postgres/ChunksDao.xml new file mode 100644 index 0000000..d7a1147 --- /dev/null +++ b/flx-ai/src/main/resources/mapper/postgres/ChunksDao.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/flx-ai/src/main/resources/mapper/postgres/FileChunksDao.xml b/flx-ai/src/main/resources/mapper/postgres/FileChunksDao.xml new file mode 100644 index 0000000..9bb9c12 --- /dev/null +++ b/flx-ai/src/main/resources/mapper/postgres/FileChunksDao.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/flx-ai/src/main/resources/mapper/postgres/GlobalFilesDao.xml b/flx-ai/src/main/resources/mapper/postgres/GlobalFilesDao.xml new file mode 100644 index 0000000..4078694 --- /dev/null +++ b/flx-ai/src/main/resources/mapper/postgres/GlobalFilesDao.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/flx-ai/src/test/java/test/com/pjilisense/flxai/controller/ChunksControllerTest.java b/flx-ai/src/test/java/test/com/pjilisense/flxai/controller/ChunksControllerTest.java new file mode 100644 index 0000000..ca27605 --- /dev/null +++ b/flx-ai/src/test/java/test/com/pjilisense/flxai/controller/ChunksControllerTest.java @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2018 人人开源 All rights reserved. + *

+ * https://www.renren.io + *

+ * 版权所有,侵权必究! + */ + +package test.com.pjilisense.flxai.controller; + +import com.alibaba.fastjson.JSON; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import test.com.pjilisense.flxai.TestUtils; + +import java.util.HashMap; +import java.util.Map; + +public class ChunksControllerTest { + @BeforeEach + public void setUpTestDataWithinTransaction() { + // set up test data within the transaction + } + @AfterEach + public void tearDownWithinTransaction() { + // execute "tear down" logic within the transaction + } + + @Test + public void testGetAllAiTools() { + String url = TestUtils.WebconnectURL+"/api/robot/chunks/search"; + String charset = TestUtils.UTF8; + try { + Map params = new HashMap(); + params.put("text","开源项目"); + params.put("userId","2f3c5337-cf1a-40ce-bbe3-ec45455d82f1"); + String body = TestUtils.doPost(url, JSON.toJSONString(params), charset); + System.out.println(body); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} \ No newline at end of file