feat:增加meilisearch

- todo:软删除问题
This commit is contained in:
3R-喜东东 2023-12-17 13:28:39 +08:00
parent 431246bc23
commit b37dfa8103
23 changed files with 468 additions and 65 deletions

18
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,18 @@
{
// 使 IntelliSense
//
// 访: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "debug 3rapp",
"request": "launch",
"runtimeArgs": ["run-script", "start:debug"],
"autoAttachChildProcesses": true,
"console": "integratedTerminal",
"runtimeExecutable": "pnpm",
"skipFiles": ["<node_internals>/**"],
"type": "node"
}
]
}

View File

@ -30,6 +30,7 @@
"deepmerge": "^4.3.1",
"fastify": "^4.24.3",
"lodash": "^4.17.21",
"meilisearch": "^0.36.0",
"mysql2": "^3.6.5",
"reflect-metadata": "^0.1.14",
"rxjs": "^7.8.1",

View File

@ -1,9 +1,5 @@
lockfileVersion: '6.0'
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
dependencies:
'@nestjs/common':
specifier: ^10.2.10
@ -35,6 +31,9 @@ dependencies:
lodash:
specifier: ^4.17.21
version: 4.17.21
meilisearch:
specifier: ^0.36.0
version: 0.36.0
mysql2:
specifier: ^3.6.5
version: 3.6.5
@ -60,7 +59,7 @@ devDependencies:
version: 10.2.1(@swc/cli@0.1.63)(@swc/core@1.3.100)
'@nestjs/schematics':
specifier: ^10.0.3
version: 10.0.3(typescript@5.3.3)
version: 10.0.3(chokidar@3.5.3)(typescript@5.2.2)
'@nestjs/testing':
specifier: ^10.2.10
version: 10.2.10(@nestjs/common@10.2.10)(@nestjs/core@10.2.10)
@ -1114,21 +1113,6 @@ packages:
- chokidar
dev: true
/@nestjs/schematics@10.0.3(typescript@5.3.3):
resolution: {integrity: sha512-2BRujK0GqGQ7j1Zpz+obVfskDnnOeVKt5aXoSaVngKo8Oczy8uYCY+R547TQB+Kf35epdfFER2pVnQrX3/It5A==}
peerDependencies:
typescript: '>=4.8.2'
dependencies:
'@angular-devkit/core': 16.2.8(chokidar@3.5.3)
'@angular-devkit/schematics': 16.2.8(chokidar@3.5.3)
comment-json: 4.2.3
jsonc-parser: 3.2.0
pluralize: 8.0.0
typescript: 5.3.3
transitivePeerDependencies:
- chokidar
dev: true
/@nestjs/swagger@7.1.16(@nestjs/common@10.2.10)(@nestjs/core@10.2.10)(class-transformer@0.5.1)(class-validator@0.14.0)(reflect-metadata@0.1.14):
resolution: {integrity: sha512-f9KBk/BX9MUKPTj7tQNYJ124wV/jP5W2lwWHLGwe/4qQXixuDOo39zP55HIJ44LE7S04B7BOeUOo9GBJD/vRcw==}
peerDependencies:
@ -2649,6 +2633,14 @@ packages:
/create-require@1.1.1:
resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==}
/cross-fetch@3.1.8:
resolution: {integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==}
dependencies:
node-fetch: 2.7.0
transitivePeerDependencies:
- encoding
dev: false
/cross-spawn@5.1.0:
resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==}
dependencies:
@ -4997,6 +4989,14 @@ packages:
tmpl: 1.0.5
dev: true
/meilisearch@0.36.0:
resolution: {integrity: sha512-swcvEYrct0/zsGj3jlbPm1OYxbH14IURnlysKlXywNicIQ5EMkSYLYCLCwOuBKAaGcdISWdgdylH9TXVLegmOQ==}
dependencies:
cross-fetch: 3.1.8
transitivePeerDependencies:
- encoding
dev: false
/memfs@3.5.3:
resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==}
engines: {node: '>= 4.0.0'}
@ -6988,3 +6988,7 @@ packages:
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
engines: {node: '>=10'}
dev: true
settings:
autoInstallPeers: true
excludeLinksFromLockfile: false

View File

@ -2,15 +2,22 @@ import { Module } from '@nestjs/common';
import { APP_FILTER, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
import { database } from '@/config';
import { content, database, meilli } from '@/config';
import { ContentModule } from '@/modules/content/content.module';
import { CoreModule } from '@/modules/core/core.module';
import { AppFilter, AppIntercepter, AppPipe } from '@/modules/core/providers';
import { DatabaseModule } from '@/modules/database/database.module';
import { MeilliModule } from '@/modules/meilisearch/melli.module';
import { WelcomeModule } from '@/modules/welcome/welcome.module';
@Module({
imports: [DatabaseModule.forRoot(database), ContentModule, WelcomeModule, CoreModule.forRoot()],
imports: [
DatabaseModule.forRoot(database),
ContentModule.forRoot(content),
WelcomeModule,
CoreModule.forRoot(),
MeilliModule.forRoot(meilli),
],
controllers: [],
providers: [
{

View File

@ -0,0 +1,5 @@
import { ContentConfig } from '@/modules/content/types';
export const content = (): ContentConfig => ({
searchType: 'meilli',
});

View File

@ -8,7 +8,7 @@ export const database = (): TypeOrmModuleOptions => ({
charset: 'utf8mb4',
logging: ['error'],
type: 'mysql',
host: '127.0.0.1',
host: 'localhost',
port: 3306,
username: 'root',
password: '12345678910',

View File

@ -1 +1,3 @@
export * from './content.config';
export * from './database.config';
export * from './meilli.config';

View File

@ -0,0 +1,9 @@
import { MelliConfig } from '@/modules/meilisearch/types';
export const meilli = (): MelliConfig => [
{
name: 'default',
host: 'http://localhost:7700',
apiKey: '12345678910',
},
];

View File

@ -18,8 +18,8 @@ const bootstrap = async () => {
fallbackOnErrors: true,
});
await app.listen(2333, () => {
console.log('api: http://localhost:2333/api');
await app.listen(3100, () => {
console.log('api: http://localhost:3100/api');
});
};

View File

@ -1,26 +1,73 @@
import { Module } from '@nestjs/common';
import { DynamicModule, Module, ModuleMetadata } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { SanitizeService } from '@/modules/content/services/sanitize.service';
import { PostSubscriber } from '@/modules/content/subscribers';
import { ContentConfig } from '@/modules/content/types';
import { DatabaseModule } from '@/modules/database/database.module';
import * as controllers from './controllers';
import * as entities from './entities';
import * as repositories from './repositories';
import * as services from './services';
import { PostService } from './services';
@Module({
imports: [
TypeOrmModule.forFeature(Object.values(entities)),
DatabaseModule.forRepository(Object.values(repositories)),
],
controllers: Object.values(controllers),
providers: [...Object.values(services), PostSubscriber, SanitizeService],
exports: [
...Object.values(services),
DatabaseModule.forRepository(Object.values(repositories)),
],
})
export class ContentModule {}
@Module({})
export class ContentModule {
static forRoot(configRegister: () => ContentConfig): DynamicModule {
const config: Required<ContentConfig> = {
searchType: 'against',
...(configRegister ? configRegister() : {}),
};
const providers: ModuleMetadata['providers'] = [
...Object.values(services),
PostSubscriber,
SanitizeService,
{
provide: PostService,
inject: [
repositories.PostRepository,
repositories.CategoryRepository,
services.CategoryService,
repositories.TagRepository,
{ token: services.SearchService, optional: true },
],
useFactory(
postRepository: repositories.PostRepository,
categoryRepository: repositories.CategoryRepository,
categoryService: services.CategoryService,
tagRepository: repositories.TagRepository,
searchService: services.SearchService,
) {
return new PostService(
postRepository,
categoryRepository,
categoryService,
tagRepository,
searchService,
config.searchType,
);
},
},
];
if (config.searchType === 'meilli') providers.push(services.SearchService);
return {
module: ContentModule,
imports: [
TypeOrmModule.forFeature(Object.values(entities)),
DatabaseModule.forRepository(Object.values(repositories)),
],
controllers: Object.values(controllers),
providers,
exports: [
...Object.values(services),
DatabaseModule.forRepository(Object.values(repositories)),
PostService,
],
};
}
}

View File

@ -71,6 +71,13 @@ export class QueryPostDto implements PaginateOptions {
@IsEnum(SelectTrashMode)
@IsOptional()
trashed?: SelectTrashMode;
@MaxLength(100, {
always: true,
message: '搜索字符串长度不得超过$constraint1',
})
@IsOptional({ always: true })
search?: string;
}
/**

View File

@ -4,6 +4,7 @@ import {
Column,
CreateDateColumn,
Entity,
Index,
ManyToOne,
PrimaryColumn,
Relation,
@ -24,6 +25,7 @@ export class CommentEntity extends BaseEntity {
@Expose()
@Column({ comment: '评论内容', type: 'text' })
@Index({ fulltext: true })
body: string;
@Expose()

View File

@ -1,5 +1,13 @@
import { Exclude, Expose, Type } from 'class-transformer';
import { Column, DeleteDateColumn, Entity, ManyToMany, PrimaryColumn, Relation } from 'typeorm';
import {
Column,
DeleteDateColumn,
Entity,
Index,
ManyToMany,
PrimaryColumn,
Relation,
} from 'typeorm';
import { PostEntity } from '@/modules/content/entities/post.entity';
@ -11,26 +19,27 @@ export class TagEntity {
id: string;
@Expose()
@Column({ comment: '标签名称' })
@Column({ comment: '分类名称' })
@Index({ fulltext: true })
name: string;
@Expose()
@Column({ comment: '标签描述', nullable: true })
description?: string;
@ManyToMany(() => PostEntity, (post) => post.tags)
posts: Relation<PostEntity[]>;
/**
* QueryBuilder生成的文章数量()
*/
@Expose()
postCount: number;
@Expose()
@Type(() => Date)
@DeleteDateColumn({
comment: '删除时间',
})
deletedAt: Date;
/**
* queryBuilder生成的文章数量()
*/
@Expose()
postCount: number;
@ManyToMany(() => PostEntity, (post) => post.tags)
posts: Relation<PostEntity[]>;
}

View File

@ -92,7 +92,7 @@ export class CategoryRepository extends TreeRepository<CategoryEntity> {
withTrashed?: boolean;
},
) {
const qb = this.createAncestorsQueryBuilder('category', 'treeClosure', entity);
const qb = this.createDescendantsQueryBuilder('category', 'treeClosure', entity);
FindOptionsUtils.applyOptionsToTreeQueryBuilder(qb, options);
qb.orderBy('category.customOrder', 'ASC');
@ -145,7 +145,7 @@ export class CategoryRepository extends TreeRepository<CategoryEntity> {
}
/**
*
*
* @param entity
* @param options
*/
@ -157,7 +157,7 @@ export class CategoryRepository extends TreeRepository<CategoryEntity> {
if (options?.withTrashed) {
qb.withDeleted();
if (options?.onlyTrashed) qb.where(`category.deleteAt IS NOT NULL`);
if (options?.onlyTrashed) qb.where(`category.deletedAt IS NOT NULL`);
}
return qb.getCount();

View File

@ -23,7 +23,6 @@ export class CategoryService {
/**
*
* @param options
*/
async findTrees(options: QueryCategoryTreeDto) {
const { trashed = SelectTrashMode.NONE } = options;

View File

@ -1,4 +1,5 @@
export * from './category.service';
export * from './comment.service';
export * from './post.service';
export * from './search.service';
export * from './tag.service';

View File

@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { isArray, isFunction, isNil, omit } from 'lodash';
import { isArray, isFunction, isNil, omit, pick } from 'lodash';
import { EntityNotFoundError, In, IsNull, Not, SelectQueryBuilder } from 'typeorm';
@ -10,6 +10,8 @@ import { PostEntity } from '@/modules/content/entities';
import { CategoryRepository, PostRepository, TagRepository } from '@/modules/content/repositories';
import { CategoryService } from '@/modules/content/services/category.service';
import { SearchService } from '@/modules/content/services/search.service';
import { SearchType } from '@/modules/content/types';
import { SelectTrashMode } from '@/modules/database/constants';
import { paginate } from '@/modules/database/helpers';
import { QueryHook } from '@/modules/database/types';
@ -25,6 +27,8 @@ export class PostService {
protected categoryRepository: CategoryRepository,
protected categoryService: CategoryService,
protected tagRepository: TagRepository,
protected searchService?: SearchService,
protected search_type: SearchType = 'against',
) {}
/**
@ -33,6 +37,12 @@ export class PostService {
* @param callback
*/
async paginate(options: QueryPostDto, callback?: QueryHook<PostEntity>) {
if (!isNil(this.searchService) && !isNil(options.search) && this.search_type === 'meilli') {
return this.searchService.search(
options.search,
pick(options, ['trashed', 'page', 'limit']),
);
}
const qb = await this.buildListQuery(this.repository.buildBaseQB(), options, callback);
return paginate(qb, options);
}
@ -69,9 +79,10 @@ export class PostService {
})
: [],
};
const item = await this.repository.save(createPostDto);
if (!isNil(this.searchService)) await this.searchService.create(item);
return this.detail(item.id);
}
@ -101,8 +112,10 @@ export class PostService {
}
await this.repository.update(data.id, omit(data, ['id', 'tags', 'category']));
const result = await this.detail(data.id);
if (!isNil(this.searchService)) await this.searchService.update([post]);
return this.detail(data.id);
return result;
}
/**
@ -111,22 +124,30 @@ export class PostService {
*/
async delete(ids: string[], trash?: boolean) {
const items = await this.repository.find({
where: { id: In(ids) } as any,
where: { id: In(ids) },
withDeleted: true,
});
let result: PostEntity[] = [];
if (trash) {
// 对已软删除的数据再次删除时直接通过remove方法从数据库中清除
const directs = items.filter((item) => !isNil(item.deletedAt));
const softs = items.filter((item) => isNil(item.deletedAt));
return [
result = [
...(await this.repository.remove(directs)),
...(await this.repository.softRemove(softs)),
];
if (!isNil(this.searchService)) {
await this.searchService.delete(directs.map(({ id }) => id));
await this.searchService.update(softs);
}
} else {
result = await this.repository.remove(items);
if (!isNil(this.searchService)) {
await this.searchService.delete(result.map(({ id }) => id));
}
}
return this.repository.remove(items);
return result;
}
/**
@ -135,14 +156,12 @@ export class PostService {
*/
async restore(ids: string[]) {
const items = await this.repository.find({
where: { id: In(ids) } as any,
where: { id: In(ids) },
withDeleted: true,
});
// 过滤掉不在回收站中的数据
const trasheds = items.filter((item) => !isNil(item)).map((item) => item.id);
if (trasheds.length < 1) return [];
await this.repository.restore(trasheds);
const qb = await this.buildListQuery(this.repository.buildBaseQB(), {}, async (qbuilder) =>
qbuilder.andWhereInIds(trasheds),
@ -179,12 +198,40 @@ export class PostService {
this.queryOrderBy(qb, orderBy);
if (category) await this.queryByCategory(category, qb);
if (!isNil(options.search)) this.buildSearchQuery(qb, options.search);
// 查询某个标签关联的文章
if (tag) qb.where('tags.id = :id', { id: tag });
if (callback) return callback(qb);
return qb;
}
protected async buildSearchQuery(qb: SelectQueryBuilder<PostEntity>, search: string) {
if (this.search_type === 'like') {
qb.andWhere('title LIKE :search', { search: `%${search}%` })
.orWhere('body LIKE :search', { search: `%${search}%` })
.orWhere('summary LIKE :search', { search: `%${search}%` })
.orWhere('category.name LIKE :search', { search: `%${search}%` })
.orWhere('tags.name LIKE :search', { search: `%${search}%` });
} else if (this.search_type === 'against') {
qb.andWhere('MATCH(title) AGAINST (:search IN BOOLEAN MODE)', {
search: `${search}*`,
})
.orWhere('MATCH(body) AGAINST (:search IN BOOLEAN MODE)', {
search: `${search}*`,
})
.orWhere('MATCH(summary) AGAINST (:search IN BOOLEAN MODE)', {
search: `${search}*`,
})
.orWhere('MATCH(category.name) AGAINST (:search IN BOOLEAN MODE)', {
search: `${search}*`,
})
.orWhere('MATCH(tags.name) AGAINST (:search IN BOOLEAN MODE)', {
search: `${search}*`,
});
}
return qb;
}
/**
* Query构建
* @param qb

View File

@ -0,0 +1,136 @@
import { ForbiddenException, Injectable } from '@nestjs/common';
import { instanceToPlain } from 'class-transformer';
import { isNil, omit, pick } from 'lodash';
import MeiliSearch from 'meilisearch';
import { PostEntity } from '@/modules/content/entities';
import { CategoryRepository, CommentRepository } from '@/modules/content/repositories';
import { SelectTrashMode } from '@/modules/database/constants';
import { MeilliService } from '@/modules/meilisearch/meilli.service';
interface SearchOption {
trashed?: SelectTrashMode;
isPublished?: boolean;
page?: number;
limit?: number;
}
async function getPostData(
catRepo: CategoryRepository,
cmtRepo: CommentRepository,
post: PostEntity,
) {
const categories = [
...(await catRepo.findAncestors(post.category)).map((item) => {
return {
id: item.id,
name: item.name,
};
}),
{ id: post.category.id, name: post.category.name },
];
const comments = (
await cmtRepo.find({
relations: ['post'],
where: { post: { id: post.id } },
})
).map((item) => ({ id: item.id, body: item.body }));
return [
{
...pick(instanceToPlain(post), [
'id',
'title',
'body',
'summary',
'commentCount',
'deletedAt',
'publishedAt',
'createdAt',
'updatedAt',
]),
categories,
tags: post.tags.map((item) => ({ id: item.id, name: item.name })),
comments,
},
];
}
@Injectable()
export class SearchService {
index = 'content';
protected _client: MeiliSearch;
constructor(
protected meilliService: MeilliService,
protected categoryRepository: CategoryRepository,
protected commentRepository: CommentRepository,
) {
this._client = this.meilliService.getClient();
}
get client() {
if (isNil(this._client)) throw new ForbiddenException('Has not any meilli search client!');
return this._client;
}
async search(text: string, param: SearchOption = {}) {
await this.client.index(this.index).addDocuments([]);
this.client.index(this.index).updateFilterableAttributes(['deletedAt', 'publishedAt']);
this.client.index(this.index).updateSortableAttributes(['updatedAt', 'commentCount']);
const option = { page: 1, limit: 10, trashed: SelectTrashMode.NONE, ...param };
const limit = isNil(option.limit) || option.limit < 1 ? 1 : option.limit;
const page = isNil(option.page) || option.page < 1 ? 1 : option.page;
let filter = ['deletedAt IS NULL'];
if (option.trashed === SelectTrashMode.ALL) {
filter = [];
} else if (option.trashed === SelectTrashMode.ONLY) {
filter = ['deletedAt IS NOT NULL'];
}
if (!isNil(option.isPublished)) {
filter.push(option.isPublished ? 'publishedAt IS NOT NULL' : 'deletedAt IS NULL');
}
const result = await this.client.index(this.index).search(text, {
page,
limit,
sort: ['updatedAt:desc', 'commentCount:desc'],
filter,
});
return {
items: result.hits,
currentPage: result.page,
perPage: result.hitsPerPage,
totalItems: result.estimatedTotalHits,
itemCount: result.totalHits,
...omit(result, ['hits', 'page', 'hitsPerPage', 'estimatedTotalHits', 'totalHits']),
};
}
async create(post: PostEntity) {
return this.client
.index(this.index)
.addDocuments(await getPostData(this.categoryRepository, this.commentRepository, post));
}
async update(posts: PostEntity[]) {
return this.client
.index(this.index)
.updateDocuments(
await Promise.all(
posts.map((post) =>
getPostData(this.categoryRepository, this.commentRepository, post),
),
),
);
}
async delete(ids: string[]) {
return this.client.index(this.index).deleteDocuments(ids);
}
}

View File

@ -0,0 +1,5 @@
export type SearchType = 'like' | 'against' | 'meilli';
export interface ContentConfig {
searchType?: SearchType;
}

View File

@ -0,0 +1,17 @@
import { MelliConfig } from '@/modules/meilisearch/types';
export const createMeilliOptions = async (
config: MelliConfig,
): Promise<MelliConfig | undefined> => {
if (config.length <= 0) return config;
let options: MelliConfig = [...config];
const names = options.map(({ name }) => name);
if (!names.includes('default')) options[0].name = 'default';
else if (names.filter((name) => name === 'default').length > 0) {
options = options.reduce(
(o, n) => (o.map(({ name }) => name).includes('default') ? o : [...o, n]),
[],
);
}
return options;
};

View File

@ -0,0 +1,52 @@
import { Injectable } from '@nestjs/common';
import { isNil } from 'lodash';
import MeiliSearch from 'meilisearch';
import { MelliConfig } from '@/modules/meilisearch/types';
@Injectable()
export class MeilliService {
protected options: MelliConfig;
/**
*
*/
protected clients: Map<string, MeiliSearch> = new Map();
constructor(options: MelliConfig) {
this.options = options;
}
getOptions() {
return this.options;
}
/**
*
*/
async createClients() {
this.options.forEach(async (o) => {
this.clients.set(o.name, new MeiliSearch(o));
});
}
/**
*
* @param name ,default
*/
getClient(name?: string): MeiliSearch {
let key = 'default';
if (!isNil(name)) key = name;
if (!this.clients.has(key)) {
throw new Error(`client ${key} does not exist`);
}
return this.clients.get(key);
}
/**
*
*/
getClients(): Map<string, MeiliSearch> {
return this.clients;
}
}

View File

@ -0,0 +1,28 @@
import { DynamicModule, Module } from '@nestjs/common';
import { createMeilliOptions } from '@/modules/meilisearch/helpers';
import { MeilliService } from '@/modules/meilisearch/meilli.service';
import { MelliConfig } from '@/modules/meilisearch/types';
@Module({})
export class MeilliModule {
static forRoot(configRegister: () => MelliConfig): DynamicModule {
return {
global: true,
module: MeilliModule,
providers: [
{
provide: MeilliService,
useFactory: async () => {
const service = new MeilliService(
await createMeilliOptions(configRegister()),
);
service.createClients();
return service;
},
},
],
exports: [MeilliService],
};
}
}

View File

@ -0,0 +1,7 @@
import { Config } from 'meilisearch';
// MelliSearch模块的配置
export type MelliConfig = MelliOption[];
// MeilliSearch的连接节点配置
export type MelliOption = Config & { name: string };