- Add POST /api/knowledge/import endpoint for .md and .zip files - Add GET /api/knowledge/export endpoint with markdown/json formats - Import parses frontmatter (title, tags, status, visibility) - Export includes frontmatter in markdown format - Add ImportExportActions component with drag-and-drop UI - Add import progress dialog with success/error summary - Add export dropdown with format selection - Include comprehensive test suite - Support bulk import with detailed error reporting
This commit is contained in:
Jason Woltje
315
apps/api/src/knowledge/services/import-export.service.ts
Normal file
315
apps/api/src/knowledge/services/import-export.service.ts
Normal file
@@ -0,0 +1,315 @@
|
||||
import { Injectable, BadRequestException } from "@nestjs/common";
|
||||
import { EntryStatus, Visibility } from "@prisma/client";
|
||||
import * as archiver from "archiver";
|
||||
import * as AdmZip from "adm-zip";
|
||||
import * as matter from "gray-matter";
|
||||
import { Readable } from "stream";
|
||||
import { PrismaService } from "../../prisma/prisma.service";
|
||||
import { KnowledgeService } from "../knowledge.service";
|
||||
import type { ExportFormat, ImportResult } from "../dto";
|
||||
import type { CreateEntryDto } from "../dto/create-entry.dto";
|
||||
|
||||
/**
|
||||
* Service for handling knowledge entry import/export operations
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImportExportService {
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly knowledgeService: KnowledgeService
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Import entries from uploaded file(s)
|
||||
* Accepts single .md file or .zip containing multiple .md files
|
||||
*/
|
||||
async importEntries(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
file: Express.Multer.File
|
||||
): Promise<{ results: ImportResult[]; totalFiles: number; imported: number; failed: number }> {
|
||||
const results: ImportResult[] = [];
|
||||
|
||||
try {
|
||||
if (file.mimetype === "text/markdown" || file.originalname.endsWith(".md")) {
|
||||
// Single markdown file
|
||||
const result = await this.importSingleMarkdown(
|
||||
workspaceId,
|
||||
userId,
|
||||
file.originalname,
|
||||
file.buffer.toString("utf-8")
|
||||
);
|
||||
results.push(result);
|
||||
} else if (
|
||||
file.mimetype === "application/zip" ||
|
||||
file.mimetype === "application/x-zip-compressed" ||
|
||||
file.originalname.endsWith(".zip")
|
||||
) {
|
||||
// Zip file containing multiple markdown files
|
||||
const zipResults = await this.importZipFile(workspaceId, userId, file.buffer);
|
||||
results.push(...zipResults);
|
||||
} else {
|
||||
throw new BadRequestException(
|
||||
"Invalid file type. Only .md and .zip files are accepted."
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new BadRequestException(
|
||||
`Failed to import file: ${error instanceof Error ? error.message : "Unknown error"}`
|
||||
);
|
||||
}
|
||||
|
||||
const imported = results.filter((r) => r.success).length;
|
||||
const failed = results.filter((r) => !r.success).length;
|
||||
|
||||
return {
|
||||
results,
|
||||
totalFiles: results.length,
|
||||
imported,
|
||||
failed,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a single markdown file
|
||||
*/
|
||||
private async importSingleMarkdown(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
filename: string,
|
||||
content: string
|
||||
): Promise<ImportResult> {
|
||||
try {
|
||||
// Parse frontmatter
|
||||
const parsed = matter(content);
|
||||
const frontmatter = parsed.data;
|
||||
const markdownContent = parsed.content.trim();
|
||||
|
||||
if (!markdownContent) {
|
||||
return {
|
||||
filename,
|
||||
success: false,
|
||||
error: "Empty content",
|
||||
};
|
||||
}
|
||||
|
||||
// Build CreateEntryDto from frontmatter and content
|
||||
const createDto: CreateEntryDto = {
|
||||
title: frontmatter.title || filename.replace(/\.md$/, ""),
|
||||
content: markdownContent,
|
||||
summary: frontmatter.summary,
|
||||
status: this.parseStatus(frontmatter.status),
|
||||
visibility: this.parseVisibility(frontmatter.visibility),
|
||||
tags: Array.isArray(frontmatter.tags) ? frontmatter.tags : undefined,
|
||||
changeNote: "Imported from markdown file",
|
||||
};
|
||||
|
||||
// Create the entry
|
||||
const entry = await this.knowledgeService.create(
|
||||
workspaceId,
|
||||
userId,
|
||||
createDto
|
||||
);
|
||||
|
||||
return {
|
||||
filename,
|
||||
success: true,
|
||||
entryId: entry.id,
|
||||
slug: entry.slug,
|
||||
title: entry.title,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
filename,
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import entries from a zip file
|
||||
*/
|
||||
private async importZipFile(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
buffer: Buffer
|
||||
): Promise<ImportResult[]> {
|
||||
const results: ImportResult[] = [];
|
||||
|
||||
try {
|
||||
const zip = new AdmZip(buffer);
|
||||
const zipEntries = zip.getEntries();
|
||||
|
||||
for (const zipEntry of zipEntries) {
|
||||
// Skip directories and non-markdown files
|
||||
if (zipEntry.isDirectory || !zipEntry.entryName.endsWith(".md")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = zipEntry.getData().toString("utf-8");
|
||||
const result = await this.importSingleMarkdown(
|
||||
workspaceId,
|
||||
userId,
|
||||
zipEntry.entryName,
|
||||
content
|
||||
);
|
||||
results.push(result);
|
||||
}
|
||||
} catch (error) {
|
||||
throw new BadRequestException(
|
||||
`Failed to extract zip file: ${error instanceof Error ? error.message : "Unknown error"}`
|
||||
);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export entries as a zip file
|
||||
*/
|
||||
async exportEntries(
|
||||
workspaceId: string,
|
||||
format: ExportFormat,
|
||||
entryIds?: string[]
|
||||
): Promise<{ stream: Readable; filename: string }> {
|
||||
// Fetch entries
|
||||
const entries = await this.fetchEntriesForExport(workspaceId, entryIds);
|
||||
|
||||
if (entries.length === 0) {
|
||||
throw new BadRequestException("No entries found to export");
|
||||
}
|
||||
|
||||
// Create archive
|
||||
const archive = archiver("zip", {
|
||||
zlib: { level: 9 },
|
||||
});
|
||||
|
||||
// Add entries to archive
|
||||
for (const entry of entries) {
|
||||
if (format === "markdown") {
|
||||
const markdown = this.entryToMarkdown(entry);
|
||||
const filename = `${entry.slug}.md`;
|
||||
archive.append(markdown, { name: filename });
|
||||
} else {
|
||||
// JSON format
|
||||
const json = JSON.stringify(entry, null, 2);
|
||||
const filename = `${entry.slug}.json`;
|
||||
archive.append(json, { name: filename });
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize archive
|
||||
archive.finalize();
|
||||
|
||||
// Generate filename
|
||||
const timestamp = new Date().toISOString().split("T")[0];
|
||||
const filename = `knowledge-export-${timestamp}.zip`;
|
||||
|
||||
return {
|
||||
stream: archive,
|
||||
filename,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch entries for export
|
||||
*/
|
||||
private async fetchEntriesForExport(
|
||||
workspaceId: string,
|
||||
entryIds?: string[]
|
||||
): Promise<any[]> {
|
||||
const where: any = { workspaceId };
|
||||
|
||||
if (entryIds && entryIds.length > 0) {
|
||||
where.id = { in: entryIds };
|
||||
}
|
||||
|
||||
const entries = await this.prisma.knowledgeEntry.findMany({
|
||||
where,
|
||||
include: {
|
||||
tags: {
|
||||
include: {
|
||||
tag: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
orderBy: {
|
||||
title: "asc",
|
||||
},
|
||||
});
|
||||
|
||||
return entries.map((entry) => ({
|
||||
id: entry.id,
|
||||
slug: entry.slug,
|
||||
title: entry.title,
|
||||
content: entry.content,
|
||||
summary: entry.summary,
|
||||
status: entry.status,
|
||||
visibility: entry.visibility,
|
||||
tags: entry.tags.map((et) => et.tag.name),
|
||||
createdAt: entry.createdAt,
|
||||
updatedAt: entry.updatedAt,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert entry to markdown format with frontmatter
|
||||
*/
|
||||
private entryToMarkdown(entry: any): string {
|
||||
const frontmatter: Record<string, any> = {
|
||||
title: entry.title,
|
||||
status: entry.status,
|
||||
visibility: entry.visibility,
|
||||
};
|
||||
|
||||
if (entry.summary) {
|
||||
frontmatter.summary = entry.summary;
|
||||
}
|
||||
|
||||
if (entry.tags && entry.tags.length > 0) {
|
||||
frontmatter.tags = entry.tags;
|
||||
}
|
||||
|
||||
frontmatter.createdAt = entry.createdAt.toISOString();
|
||||
frontmatter.updatedAt = entry.updatedAt.toISOString();
|
||||
|
||||
// Build frontmatter string
|
||||
const frontmatterStr = Object.entries(frontmatter)
|
||||
.map(([key, value]) => {
|
||||
if (Array.isArray(value)) {
|
||||
return `${key}:\n - ${value.join("\n - ")}`;
|
||||
}
|
||||
return `${key}: ${value}`;
|
||||
})
|
||||
.join("\n");
|
||||
|
||||
return `---\n${frontmatterStr}\n---\n\n${entry.content}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse status from frontmatter
|
||||
*/
|
||||
private parseStatus(value: any): EntryStatus | undefined {
|
||||
if (!value) return undefined;
|
||||
const statusMap: Record<string, EntryStatus> = {
|
||||
DRAFT: EntryStatus.DRAFT,
|
||||
PUBLISHED: EntryStatus.PUBLISHED,
|
||||
ARCHIVED: EntryStatus.ARCHIVED,
|
||||
};
|
||||
return statusMap[String(value).toUpperCase()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse visibility from frontmatter
|
||||
*/
|
||||
private parseVisibility(value: any): Visibility | undefined {
|
||||
if (!value) return undefined;
|
||||
const visibilityMap: Record<string, Visibility> = {
|
||||
PRIVATE: Visibility.PRIVATE,
|
||||
WORKSPACE: Visibility.WORKSPACE,
|
||||
PUBLIC: Visibility.PUBLIC,
|
||||
};
|
||||
return visibilityMap[String(value).toUpperCase()];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user