17 lines
550 B
TypeScript
17 lines
550 B
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import { chunkMarkdown } from "./internal.js";
|
|
|
|
describe("chunkMarkdown", () => {
|
|
it("splits overly long lines into max-sized chunks", () => {
|
|
const chunkTokens = 400;
|
|
const maxChars = chunkTokens * 4;
|
|
const content = "a".repeat(maxChars * 3 + 25);
|
|
const chunks = chunkMarkdown(content, { tokens: chunkTokens, overlap: 0 });
|
|
expect(chunks.length).toBeGreaterThan(1);
|
|
for (const chunk of chunks) {
|
|
expect(chunk.text.length).toBeLessThanOrEqual(maxChars);
|
|
}
|
|
});
|
|
});
|