知识库检索

flx-ai/pom.xml

@@ -148,6 +148,12 @@
             <artifactId>knife4j-spring-boot-starter</artifactId>
             <version>${knife4j.version}</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/com.hankcs/hanlp -->
+        <dependency>
+            <groupId>com.hankcs</groupId>
+            <artifactId>hanlp</artifactId>
+            <version>portable-1.8.6</version>
+        </dependency>
     </dependencies>
     <!-- 阿里云maven仓库 -->
     <repositories>

flx-ai/src/main/java/com/pjilisense/flxai/controller/ChunksController.java

@@ -0,0 +1,93 @@
+package com.pjilisense.flxai.controller;
+
+import com.pjilisense.flxai.base.constant.Constant;
+import com.pjilisense.flxai.dto.FilesDTO;
+import com.pjilisense.flxai.page.PageData;
+import com.pjilisense.flxai.utils.Result;
+import com.pjilisense.flxai.dto.ChunksDTO;
+import com.pjilisense.flxai.service.ChunksService;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import springfox.documentation.annotations.ApiIgnore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@RestController
+@RequestMapping(Constant.BASE_PATH+"chunks")
+@Api(tags="${comments}")
+public class ChunksController {
+
+    private static Logger logger = LoggerFactory.getLogger(ChunksController.class);
+
+    @Autowired
+    private ChunksService chunksService;
+
+    @GetMapping("page")
+    @ApiOperation("分页")
+    @ApiImplicitParams({
+        @ApiImplicitParam(name = Constant.PAGE, value = "当前页码，从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) ,
+        @ApiImplicitParam(name = Constant.ORDER, value = "排序方式，可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class)
+    })
+    public Result<PageData<ChunksDTO>> page(@ApiIgnore @RequestParam Map<String, Object> params){
+        PageData<ChunksDTO> page = chunksService.page(params);
+
+        return new Result<PageData<ChunksDTO>>().ok(page);
+    }
+
+    @GetMapping("{id}")
+    @ApiOperation("信息")
+    public Result<ChunksDTO> get(@PathVariable("id") Long id){
+        ChunksDTO data = chunksService.get(id);
+
+        return new Result<ChunksDTO>().ok(data);
+    }
+    @PostMapping("/search")
+    @ApiOperation("模糊检索")
+    public Result searchByText(@RequestBody ChunksDTO dto){
+        List<Map<String,Object>> list =chunksService.search(dto.getUserId(),dto.getText());
+        return new Result().ok(list);
+    }
+    @PostMapping
+    @ApiOperation("保存")
+    public Result save(@RequestBody ChunksDTO dto){
+        chunksService.save(dto);
+
+        return new Result();
+    }
+
+    @PutMapping
+    @ApiOperation("修改")
+    public Result update(@RequestBody ChunksDTO dto){
+
+        chunksService.update(dto);
+
+        return new Result();
+    }
+
+    @DeleteMapping
+    @ApiOperation("删除")
+    public Result delete(@RequestBody Long[] ids){
+        //效验数据
+        chunksService.delete(ids);
+
+        return new Result();
+    }
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/controller/FileChunksController.java

@@ -0,0 +1,88 @@
+package com.pjilisense.flxai.controller;
+
+import com.pjilisense.flxai.base.constant.Constant;
+import com.pjilisense.flxai.page.PageData;
+import com.pjilisense.flxai.utils.Result;
+import com.pjilisense.flxai.dto.FileChunksDTO;
+import com.pjilisense.flxai.service.FileChunksService;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import springfox.documentation.annotations.ApiIgnore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@RestController
+@RequestMapping(Constant.BASE_PATH+"filechunks")
+@Api(tags="${comments}")
+public class FileChunksController {
+
+    private static Logger logger = LoggerFactory.getLogger(FileChunksController.class);
+
+    @Autowired
+    private FileChunksService fileChunksService;
+
+    @GetMapping("page")
+    @ApiOperation("分页")
+    @ApiImplicitParams({
+        @ApiImplicitParam(name = Constant.PAGE, value = "当前页码，从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) ,
+        @ApiImplicitParam(name = Constant.ORDER, value = "排序方式，可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class)
+    })
+    public Result<PageData<FileChunksDTO>> page(@ApiIgnore @RequestParam Map<String, Object> params){
+        PageData<FileChunksDTO> page = fileChunksService.page(params);
+
+        return new Result<PageData<FileChunksDTO>>().ok(page);
+    }
+
+    @GetMapping("{id}")
+    @ApiOperation("信息")
+    public Result<FileChunksDTO> get(@PathVariable("id") Long id){
+        FileChunksDTO data = fileChunksService.get(id);
+
+        return new Result<FileChunksDTO>().ok(data);
+    }
+
+    @PostMapping
+    @ApiOperation("保存")
+    public Result save(@RequestBody FileChunksDTO dto){
+
+        fileChunksService.save(dto);
+
+        return new Result();
+    }
+
+    @PutMapping
+    @ApiOperation("修改")
+    public Result update(@RequestBody FileChunksDTO dto){
+
+        fileChunksService.update(dto);
+
+        return new Result();
+    }
+
+    @DeleteMapping
+    @ApiOperation("删除")
+    public Result delete(@RequestBody Long[] ids){
+
+        fileChunksService.delete(ids);
+
+        return new Result();
+    }
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/controller/GlobalFilesController.java

@@ -0,0 +1,88 @@
+package com.pjilisense.flxai.controller;
+
+import com.pjilisense.flxai.base.constant.Constant;
+import com.pjilisense.flxai.page.PageData;
+import com.pjilisense.flxai.utils.Result;
+import com.pjilisense.flxai.dto.GlobalFilesDTO;
+import com.pjilisense.flxai.service.GlobalFilesService;
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiImplicitParam;
+import io.swagger.annotations.ApiImplicitParams;
+import io.swagger.annotations.ApiOperation;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+import springfox.documentation.annotations.ApiIgnore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.http.HttpServletResponse;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@RestController
+@RequestMapping(Constant.BASE_PATH+"globalfiles")
+@Api(tags="${comments}")
+public class GlobalFilesController {
+
+    private static Logger logger = LoggerFactory.getLogger(GlobalFilesController.class);
+
+    @Autowired
+    private GlobalFilesService globalFilesService;
+
+    @GetMapping("page")
+    @ApiOperation("分页")
+    @ApiImplicitParams({
+        @ApiImplicitParam(name = Constant.PAGE, value = "当前页码，从1开始", paramType = "query", required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.LIMIT, value = "每页显示记录数", paramType = "query",required = true, dataType="Integer", dataTypeClass =Integer.class) ,
+        @ApiImplicitParam(name = Constant.ORDER_FIELD, value = "排序字段", paramType = "query", dataType="String", dataTypeClass =String.class) ,
+        @ApiImplicitParam(name = Constant.ORDER, value = "排序方式，可选值(asc、desc)", paramType = "query", dataType="String", dataTypeClass =String.class)
+    })
+    public Result<PageData<GlobalFilesDTO>> page(@ApiIgnore @RequestParam Map<String, Object> params){
+        PageData<GlobalFilesDTO> page = globalFilesService.page(params);
+
+        return new Result<PageData<GlobalFilesDTO>>().ok(page);
+    }
+
+    @GetMapping("{id}")
+    @ApiOperation("信息")
+    public Result<GlobalFilesDTO> get(@PathVariable("id") Long id){
+        GlobalFilesDTO data = globalFilesService.get(id);
+
+        return new Result<GlobalFilesDTO>().ok(data);
+    }
+
+    @PostMapping
+    @ApiOperation("保存")
+    public Result save(@RequestBody GlobalFilesDTO dto){
+
+        globalFilesService.save(dto);
+
+        return new Result();
+    }
+
+    @PutMapping
+    @ApiOperation("修改")
+    public Result update(@RequestBody GlobalFilesDTO dto){
+
+        globalFilesService.update(dto);
+
+        return new Result();
+    }
+
+    @DeleteMapping
+    @ApiOperation("删除")
+    public Result delete(@RequestBody Long[] ids){
+
+        globalFilesService.delete(ids);
+
+        return new Result();
+    }
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/dao/ChunksDao.java

@@ -0,0 +1,20 @@
+package com.pjilisense.flxai.dao;
+
+import com.pjilisense.flxai.base.dao.BaseDao;
+import com.pjilisense.flxai.entity.ChunksEntity;
+import org.apache.ibatis.annotations.Mapper;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Mapper
+public interface ChunksDao extends BaseDao<ChunksEntity> {
+    List<Map<String,Object>> queryCsv(Map<String, Object> map);
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/dao/FileChunksDao.java

@@ -0,0 +1,16 @@
+package com.pjilisense.flxai.dao;
+
+import com.pjilisense.flxai.base.dao.BaseDao;
+import com.pjilisense.flxai.entity.FileChunksEntity;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Mapper
+public interface FileChunksDao extends BaseDao<FileChunksEntity> {
+	
+}

flx-ai/src/main/java/com/pjilisense/flxai/dao/GlobalFilesDao.java

@@ -0,0 +1,16 @@
+package com.pjilisense.flxai.dao;
+
+import com.pjilisense.flxai.base.dao.BaseDao;
+import com.pjilisense.flxai.entity.GlobalFilesEntity;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Mapper
+public interface GlobalFilesDao extends BaseDao<GlobalFilesEntity> {
+	
+}

flx-ai/src/main/java/com/pjilisense/flxai/dto/ChunksDTO.java

@@ -0,0 +1,64 @@
+package com.pjilisense.flxai.dto;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@ApiModel(value = "${comments}")
+public class ChunksDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+	@ApiModelProperty(value = "$column.comments")
+	private UUID id;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String text;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String abstractx;
+
+	@ApiModelProperty(value = "$column.comments")
+	private Map<String,Object> metadata;
+
+	@ApiModelProperty(value = "$column.comments")
+	private Integer index;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String type;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime createdAt;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime updatedAt;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String userId;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime accessedAt;
+
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/dto/FileChunksDTO.java

@@ -0,0 +1,38 @@
+package com.pjilisense.flxai.dto;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.UUID;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@ApiModel(value = "${comments}")
+public class FileChunksDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String fileId;
+
+	@ApiModelProperty(value = "$column.comments")
+	private UUID chunkId;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime createdAt;
+
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/dto/GlobalFilesDTO.java

@@ -0,0 +1,52 @@
+package com.pjilisense.flxai.dto;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.Map;
+
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@ApiModel(value = "${comments}")
+public class GlobalFilesDTO implements Serializable {
+    private static final long serialVersionUID = 1L;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String hashId;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String fileType;
+
+	@ApiModelProperty(value = "$column.comments")
+	private Integer size;
+
+	@ApiModelProperty(value = "$column.comments")
+	private String url;
+
+	@ApiModelProperty(value = "$column.comments")
+	private Map<String,Object> metadata;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime createdAt;
+
+	@ApiModelProperty(value = "$column.comments")
+	@JsonSerialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeSerializer.class)
+	@JsonDeserialize(using = com.pjilisense.flxai.wrapper.OffsetDateTimeDeserializer.class)
+	private OffsetDateTime accessedAt;
+
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/entity/ChunksEntity.java

@@ -0,0 +1,67 @@
+package com.pjilisense.flxai.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.pjilisense.flxai.handle.JsonbTypeHandler;
+import com.pjilisense.flxai.handle.UUIDTypeHandler;
+import lombok.Data;
+
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@TableName("chunks")
+public class ChunksEntity {
+
+    /**
+     * $column.comments
+     */
+	@TableField(typeHandler = UUIDTypeHandler.class)
+	private UUID id;
+    /**
+     * $column.comments
+     */
+	private String text;
+    /**
+     * $column.comments
+     */
+	@TableField(value="abstract")
+	private String abstractx;
+    /**
+     * $column.comments
+     */
+	@TableField(value = "metadata", typeHandler = JsonbTypeHandler.class)
+	private Map<String,Object> metadata;
+    /**
+     * $column.comments
+     */
+	private Integer index;
+    /**
+     * $column.comments
+     */
+	private String type;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime createdAt;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime updatedAt;
+    /**
+     * $column.comments
+     */
+	private String userId;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime accessedAt;
+}

flx-ai/src/main/java/com/pjilisense/flxai/entity/FileChunksEntity.java

@@ -0,0 +1,35 @@
+package com.pjilisense.flxai.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.pjilisense.flxai.handle.UUIDTypeHandler;
+import lombok.Data;
+
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.UUID;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@TableName("file_chunks")
+public class FileChunksEntity {
+
+    /**
+     * $column.comments
+     */
+	private String fileId;
+    /**
+     * $column.comments
+     */
+    @TableField(typeHandler = UUIDTypeHandler.class)
+	private UUID chunkId;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime createdAt;
+}

flx-ai/src/main/java/com/pjilisense/flxai/entity/GlobalFilesEntity.java

@@ -0,0 +1,51 @@
+package com.pjilisense.flxai.entity;
+
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.pjilisense.flxai.handle.JsonbTypeHandler;
+import lombok.Data;
+
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Data
+@TableName("global_files")
+public class GlobalFilesEntity {
+
+    /**
+     * $column.comments
+     */
+	private String hashId;
+    /**
+     * $column.comments
+     */
+	private String fileType;
+    /**
+     * $column.comments
+     */
+	private Integer size;
+    /**
+     * $column.comments
+     */
+	private String url;
+    /**
+     * $column.comments
+     */
+    @TableField(value = "metadata", typeHandler = JsonbTypeHandler.class)
+	private Map<String,Object> metadata;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime createdAt;
+    /**
+     * $column.comments
+     */
+	private OffsetDateTime accessedAt;
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/ChunksService.java

@@ -0,0 +1,19 @@
+package com.pjilisense.flxai.service;
+
+import com.pjilisense.flxai.base.service.CrudService;
+import com.pjilisense.flxai.dto.ChunksDTO;
+import com.pjilisense.flxai.entity.ChunksEntity;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+public interface ChunksService extends CrudService<ChunksEntity, ChunksDTO> {
+
+    List<Map<String,Object>> search(String userid,String text);
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/FileChunksService.java

@@ -0,0 +1,15 @@
+package com.pjilisense.flxai.service;
+
+import com.pjilisense.flxai.base.service.CrudService;
+import com.pjilisense.flxai.dto.FileChunksDTO;
+import com.pjilisense.flxai.entity.FileChunksEntity;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+public interface FileChunksService extends CrudService<FileChunksEntity, FileChunksDTO> {
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/GlobalFilesService.java

@@ -0,0 +1,15 @@
+package com.pjilisense.flxai.service;
+
+import com.pjilisense.flxai.base.service.CrudService;
+import com.pjilisense.flxai.dto.GlobalFilesDTO;
+import com.pjilisense.flxai.entity.GlobalFilesEntity;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+public interface GlobalFilesService extends CrudService<GlobalFilesEntity, GlobalFilesDTO> {
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/impl/ChunksServiceImpl.java

@@ -0,0 +1,244 @@
+package com.pjilisense.flxai.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.seg.Segment;
+import com.hankcs.hanlp.seg.common.Term;
+import com.pjilisense.flxai.base.service.impl.CrudServiceImpl;
+import com.pjilisense.flxai.dao.ChunksDao;
+import com.pjilisense.flxai.dto.ChunksDTO;
+import com.pjilisense.flxai.entity.ChunksEntity;
+import com.pjilisense.flxai.service.ChunksService;
+import cn.hutool.core.util.StrUtil;
+import org.springframework.stereotype.Service;
+
+import java.math.BigDecimal;
+import java.util.*;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Service
+public class ChunksServiceImpl extends CrudServiceImpl<ChunksDao, ChunksEntity, ChunksDTO> implements ChunksService {
+
+    @Override
+    public QueryWrapper<ChunksEntity> getWrapper(Map<String, Object> params){
+        String id = (String)params.get("id");
+
+        QueryWrapper<ChunksEntity> wrapper = new QueryWrapper<>();
+        wrapper.eq(StrUtil.isNotBlank(id), "id", id);
+
+        return wrapper;
+    }
+
+
+    @Override
+    public List<Map<String, Object>> search(String userid,String keyText) {
+        Segment seg= HanLP.newSegment();
+        seg.enableCustomDictionary(false);
+//        String keyText="我爱北京天安门";
+//        keyText="HanLP是由一系列NLP工具组成的开源项目，其中包含了中文分词、词性标注、命名实体识别等功能。";
+        List<Term> lst = seg.seg(keyText);
+        ArrayList<String> listWord=processTermList(lst);
+        //ArrayList<String> listSentence=processTermListx(lst);
+        LinkedHashSet<String> set = new LinkedHashSet<>();
+        set.addAll(listWord);
+        //set.addAll(listSentence);
+        listWord.clear();
+        //listSentence.clear();
+        ArrayList<String> wordList=processTermList(lst);
+        wordList.addAll(set);
+        set.clear();
+        HashMap<String,Object> map = new HashMap<>();
+        map.put("userid",userid);
+        map.put("wordList",wordList);
+        List<Map<String,Object>> list = baseDao.queryCsv(map);
+        List<Map<String,Object>> listRet = new ArrayList<>() ;
+        if(list!=null && list.size()>0) {
+            ArrayList<String> listText=processTermListN0Com(lst);
+            List<String> idList = new ArrayList<>();
+            for (Map<String, Object> mp: list){
+                String ttext = String.valueOf(mp.get("text"));
+                int pos = ttext.indexOf(',');
+                String question = ttext.substring(0,pos);
+                String answer = ttext.substring(pos+1);
+                mp.put("question",question);
+                mp.put("answer",answer);
+                List<Term> lst2 = seg.seg(question);
+                ArrayList<String> listQuestion=processTermListN0Com(lst2);
+                double similarity =calculateSimilarity(listText, listQuestion);
+                BigDecimal bd = new BigDecimal(similarity);
+                if(bd.compareTo(BigDecimal.ZERO.stripTrailingZeros())==0){
+                    idList.add(String.valueOf(mp.get("id")));
+                } else {
+                    System.out.println(bd.toString());
+                    mp.put("similarity",bd.toString());
+                }
+            }
+            for(Map<String, Object> mp: list){
+                String id=String.valueOf(mp.get("id"));
+                if(!idList.contains(id)){
+                    listRet.add(mp);
+                }
+            }
+            list.clear();
+            list.addAll(listRet);
+            Collections.sort(list, new Comparator<Map<String, Object>>() {
+                @Override
+                public int compare(Map<String, Object> o1, Map<String, Object> o2) {
+                    BigDecimal bd1 = new BigDecimal(String.valueOf(o1.get("similarity")));
+                    BigDecimal bd2 = new BigDecimal(String.valueOf(o2.get("similarity")));
+                    return bd1.compareTo(bd2)*(-1);
+                }
+            });
+        }
+        return list;
+    }
+
+
+    private static ArrayList<String> processTermList(List<Term> lst) {
+        ArrayList<String> ret = new ArrayList<>();
+        StringBuilder sd = new StringBuilder();
+        for(int i=0;i<lst.size();i++) {
+            Term term = lst.get(i);
+            if(term.nature.startsWith('w')) {
+                if(sd.length()>0) {
+                    ret.add(sd.toString());
+                    sd = sd.delete(0,sd.length());
+                }
+                continue;
+            }else {
+                String word = term.word;
+                if(word.length()==1) {
+                    sd.append(word);
+                    if(i+1< lst.size()) {
+                        Term termx = lst.get(i+1);
+                        if(!termx.nature.startsWith('w')) {
+                            String wordx = termx.word;
+                            sd.append(wordx);
+                            if(wordx.length()>1) {
+                                ret.add(sd.toString());
+                                sd = sd.delete(0,sd.length());
+                            }
+                            i++;
+                        }
+                    }
+                } else {
+                    sd.append(word);
+                    ret.add(sd.toString());
+                    sd = sd.delete(0,sd.length());
+                }
+            }
+
+        }
+//        System.out.println("ret=");
+//        for(String term : ret) {
+//            System.out.println(term);
+//        }
+        return ret;
+    }
+
+    private static ArrayList<String> processTermListN0Com(List<Term> lst) {
+        ArrayList<String> ret = new ArrayList<>();
+        StringBuilder sd = new StringBuilder();
+        for(int i=0;i<lst.size();i++) {
+            Term term = lst.get(i);
+            if(term.nature.startsWith('w')) {
+                if(sd.length()>0) {
+                    ret.add(sd.toString());
+                    sd = sd.delete(0,sd.length());
+                }
+                continue;
+            }else {
+                String word = term.word;
+                if(word.length()==1) {
+                    sd.append(word);
+                    if(i+1< lst.size()) {
+                        Term termx = lst.get(i+1);
+                        if(!termx.nature.startsWith('w')) {
+                            String wordx = termx.word;
+                            sd.append(wordx);
+                            if(wordx.length()>0) {
+                                ret.add(sd.toString());
+                                sd = sd.delete(0,sd.length());
+                            }
+                            i++;
+                        }
+                    }
+                } else {
+                    sd.append(word);
+                    ret.add(sd.toString());
+                    sd = sd.delete(0,sd.length());
+                }
+            }
+
+        }
+//        System.out.println("ret=");
+//        for(String term : ret) {
+//            System.out.println(term);
+//        }
+        return ret;
+    }
+    private static ArrayList<String> processTermListx(List<Term> lst) {
+        ArrayList<String> ret = new ArrayList<>();
+        StringBuilder sd = new StringBuilder();
+        for(int i=0;i<lst.size();i++) {
+            Term term = lst.get(i);
+            if(term.nature.startsWith('w')) {
+                if(sd.length()>0) {
+                    ret.add(sd.toString());
+                    sd = sd.delete(0,sd.length());
+                }
+                continue;
+            }else {
+                String word = term.word;
+                sd.append(word);
+            }
+        }
+//        System.out.println("ret=");
+//        for(String term : ret) {
+//            System.out.println(term);
+//        }
+        return ret;
+    }
+
+    // 创建单词频率映射
+    private Map<String, Integer> createFrequencyMap(List<String> words) {
+        Map<String, Integer> freqMap = new HashMap<>();
+        for (String word : words) {
+            freqMap.put(word, freqMap.getOrDefault(word, 0) + 1);
+        }
+        return freqMap;
+    }
+
+    // 计算余弦相似度
+    private double calculateSimilarity(List<String> words1, List<String> words2) {
+//        List<String> words1 = preprocessText(text1);
+//        List<String> words2 = preprocessText(text2);
+
+        Map<String, Integer> freqMap1 = createFrequencyMap(words1);
+        Map<String, Integer> freqMap2 = createFrequencyMap(words2);
+
+        double dotProduct = 0.0;
+        double magnitude1 = 0.0;
+        double magnitude2 = 0.0;
+
+        for (String word : freqMap1.keySet()) {
+            int freq1 = freqMap1.get(word);
+            magnitude1 += freq1 * freq1;
+            if (freqMap2.containsKey(word)) {
+                int freq2 = freqMap2.get(word);
+                dotProduct += freq1 * freq2;
+            }
+        }
+
+        for (int freq : freqMap2.values()) {
+            magnitude2 += freq * freq;
+        }
+
+        return dotProduct / (Math.sqrt(magnitude1) * Math.sqrt(magnitude2));
+    }
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/impl/FileChunksServiceImpl.java

@@ -0,0 +1,34 @@
+package com.pjilisense.flxai.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.pjilisense.flxai.base.service.impl.CrudServiceImpl;
+import com.pjilisense.flxai.dao.FileChunksDao;
+import com.pjilisense.flxai.dto.FileChunksDTO;
+import com.pjilisense.flxai.entity.FileChunksEntity;
+import com.pjilisense.flxai.service.FileChunksService;
+import cn.hutool.core.util.StrUtil;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Service
+public class FileChunksServiceImpl extends CrudServiceImpl<FileChunksDao, FileChunksEntity, FileChunksDTO> implements FileChunksService {
+
+    @Override
+    public QueryWrapper<FileChunksEntity> getWrapper(Map<String, Object> params){
+        String id = (String)params.get("id");
+
+        QueryWrapper<FileChunksEntity> wrapper = new QueryWrapper<>();
+        wrapper.eq(StrUtil.isNotBlank(id), "id", id);
+
+        return wrapper;
+    }
+
+
+}

flx-ai/src/main/java/com/pjilisense/flxai/service/impl/GlobalFilesServiceImpl.java

@@ -0,0 +1,34 @@
+package com.pjilisense.flxai.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+import com.pjilisense.flxai.base.service.impl.CrudServiceImpl;
+import com.pjilisense.flxai.dao.GlobalFilesDao;
+import com.pjilisense.flxai.dto.GlobalFilesDTO;
+import com.pjilisense.flxai.entity.GlobalFilesEntity;
+import com.pjilisense.flxai.service.GlobalFilesService;
+import cn.hutool.core.util.StrUtil;
+import org.springframework.stereotype.Service;
+
+import java.util.Map;
+
+/**
+ * ${comments}
+ *
+ * @author liushujing liushujing@philisense.com
+ * @since 1.0.0 2025-02-10
+ */
+@Service
+public class GlobalFilesServiceImpl extends CrudServiceImpl<GlobalFilesDao, GlobalFilesEntity, GlobalFilesDTO> implements GlobalFilesService {
+
+    @Override
+    public QueryWrapper<GlobalFilesEntity> getWrapper(Map<String, Object> params){
+        String id = (String)params.get("id");
+
+        QueryWrapper<GlobalFilesEntity> wrapper = new QueryWrapper<>();
+        wrapper.eq(StrUtil.isNotBlank(id), "id", id);
+
+        return wrapper;
+    }
+
+
+}

flx-ai/src/main/resources/mapper/postgres/ChunksDao.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.pjilisense.flxai.dao.ChunksDao">
+
+    <resultMap type="com.pjilisense.flxai.entity.ChunksEntity" id="chunksMap">
+        <result property="id" column="id" typeHandler="com.pjilisense.flxai.handle.UUIDTypeHandler"/>
+        <result property="text" column="text"/>
+        <result property="abstractx" column="abstract"/>
+        <result property="metadata" column="metadata" typeHandler="com.pjilisense.flxai.handle.JsonbTypeHandler"/>
+        <result property="index" column="index"/>
+        <result property="type" column="type"/>
+        <result property="createdAt" column="created_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime" />
+        <result property="updatedAt" column="updated_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime" />
+        <result property="userId" column="user_id"/>
+        <result property="accessedAt" column="accessed_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime"/>
+    </resultMap>
+
+    <resultMap type="java.util.Map" id="BaseResultMap">
+        <result property="id" column="id" typeHandler="com.pjilisense.flxai.handle.UUIDTypeHandler"/>
+        <result property="text" column="text"/>
+        <result property="fid" column="fid"/>
+        <result property="fname" column="fname" />
+    </resultMap>
+
+    <select id="queryCsv" resultMap="BaseResultMap" parameterType="java.util.Map">
+        select a.id ,a."text" ,f.id as fid ,f."name" as fname from chunks a left join file_chunks fc on fc.chunk_id =a.id
+        left join files f on f.id =fc.file_id
+        where f.file_type ='text/plain' and a.user_id =#{userid}
+        <if test="wordList != null">
+            AND
+            <foreach collection="wordList" item="word" index="index" open="(" close=")" separator="or">
+                a."text" like '%${word}%'
+            </foreach>
+        </if>
+    </select>
+
+</mapper>

flx-ai/src/main/resources/mapper/postgres/FileChunksDao.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.pjilisense.flxai.dao.FileChunksDao">
+
+    <resultMap type="com.pjilisense.flxai.entity.FileChunksEntity" id="fileChunksMap">
+        <result property="fileId" column="file_id"/>
+        <result property="chunkId" column="chunk_id" typeHandler="com.pjilisense.flxai.handle.UUIDTypeHandler"/>
+        <result property="createdAt" column="created_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime"/>
+    </resultMap>
+
+
+</mapper>

flx-ai/src/main/resources/mapper/postgres/GlobalFilesDao.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.pjilisense.flxai.dao.GlobalFilesDao">
+
+    <resultMap type="com.pjilisense.flxai.entity.GlobalFilesEntity" id="globalFilesMap">
+        <result property="hashId" column="hash_id"/>
+        <result property="fileType" column="file_type"/>
+        <result property="size" column="size"/>
+        <result property="url" column="url"/>
+        <result property="metadata" column="metadata" typeHandler="com.pjilisense.flxai.handle.JsonbTypeHandler"/>
+        <result property="createdAt" column="created_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime"/>
+        <result property="accessedAt" column="accessed_at" jdbcType="TIMESTAMP_WITH_TIMEZONE" javaType="java.time.OffsetDateTime"/>
+    </resultMap>
+
+
+</mapper>

flx-ai/src/test/java/test/com/pjilisense/flxai/controller/ChunksControllerTest.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2018 人人开源 All rights reserved.
+ * <p>
+ * https://www.renren.io
+ * <p>
+ * 版权所有，侵权必究！
+ */
+
+package test.com.pjilisense.flxai.controller;
+
+import com.alibaba.fastjson.JSON;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import test.com.pjilisense.flxai.TestUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ChunksControllerTest {
+    @BeforeEach
+    public void setUpTestDataWithinTransaction() {
+        // set up test data within the transaction
+    }
+    @AfterEach
+    public void tearDownWithinTransaction() {
+        // execute "tear down" logic within the transaction
+    }
+
+    @Test
+    public void testGetAllAiTools() {
+        String url = TestUtils.WebconnectURL+"/api/robot/chunks/search";
+        String charset = TestUtils.UTF8;
+        try {
+            Map<String, Object> params = new HashMap<String, Object>();
+            params.put("text","开源项目");
+            params.put("userId","2f3c5337-cf1a-40ce-bbe3-ec45455d82f1");
+            String body = TestUtils.doPost(url, JSON.toJSONString(params), charset);
+            System.out.println(body);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+}