| 1 | import os |
| 2 | import logging |
| 3 | import asyncio |
| 4 | from typing import List, Tuple |
| 5 | from langchain_core.messages import HumanMessage, SystemMessage |
| 6 | from langchain.chat_models import init_chat_model |
| 7 | from langchain.text_splitter import RecursiveCharacterTextSplitter |
| 8 | |
| 9 | |
| 10 | |
| 11 | system_prompt_template_compress_novel_chunk = \ |
| 12 | """ |
| 13 | You are an expert text compression assistant specialized in literary content. Your goal is to condense novels or story excerpts while preserving core narrative elements, key details, character development, and plot coherence. |
| 14 | |
| 15 | |
| 16 | **TASK** |
| 17 | Compress the provided input text to reduce its length significantly, eliminating redundancies, overly descriptive passages, and minor details—but without losing essential story arcs, dialogue, or emotional impact. Aim for clarity and readability in the compressed output. |
| 18 | |
| 19 | |
| 20 | **INPUT** |
| 21 | A segment of a novel (possibly truncated due to context length constraints). It is enclosed within <NOVEL_CHUNK_START> and <NOVEL_CHUNK_END> tags. |
| 22 | |
| 23 | |
| 24 | **OUTPUT** |
| 25 | A compressed version of the input text, retaining the core narrative, critical events, and character interactions. |
| 26 | |
| 27 | **GUIDELINES** |
| 28 | 1. Fidelity to the Plot: Absolutely preserve all major plot points, twists, revelations, and the sequence of key events. Do not omit crucial story elements. |
| 29 | 2. Character Consistency: Maintain character actions, decisions, and development. Important dialogue that reveals plot or character can be condensed or paraphrased but its meaning must be kept intact. |
| 30 | 3. Streamline Description: Reduce lengthy descriptions of settings, characters, or objects to their most essential and evocative elements. Capture the mood and critical details without the elaborate prose. |
| 31 | 4. Condense Internal Monologue: Paraphrase characters' extended internal thoughts and reflections, focusing on the key realizations or decisions they lead to. |
| 32 | 5. Simplify Language: Use more direct and concise language. Combine sentences, eliminate redundant adverbs and adjectives, and avoid repetitive phrasing. |
| 33 | 6. Cohesion and Flow: Ensure the compressed text is smooth, readable, and maintains a logical narrative flow. It should not feel like a fragmented list of events. |
| 34 | 7. Discard any non-narrative text (e.g., "Please follow my account!", "Background setting:...", personal opinions). |
| 35 | 8. Produce a seamless paragraph (or paragraphs if necessary) without markers (e.g., "Chapter 1") or section breaks. |
| 36 | 9. The language of output should be consistent with the original text. |
| 37 | """ |
| 38 | |
| 39 | human_prompt_template_compress_novel_chunk = \ |
| 40 | """ |
| 41 | <NOVEL_CHUNK_START> |
| 42 | {novel_chunk} |
| 43 | <NOVEL_CHUNK_END> |
| 44 | """ |
| 45 | |
| 46 | |
| 47 | system_prompt_template_aggregate = \ |
| 48 | """ |
| 49 | You are a professional text processing assistant specializing in the aggregation and refinement of segmented text chunks. Your expertise lies in seamlessly merging sequential text fragments while intelligently handling overlapping or duplicated content expressed in different ways. |
| 50 | |
| 51 | **TASK** |
| 52 | Aggregate the provided text chunks into a coherent and continuous short story. Carefully identify and resolve overlaps where the end of one chunk and the beginning of the next chunk contain semantically similar content but with different expressions. Remove redundant repetitions while preserving the original meaning, style, and flow of the text. Ensure all non-overlapping content remains unchanged and intact. |
| 53 | |
| 54 | |
| 55 | **INPUT** |
| 56 | A sequence of text chunks (ordered from first to last), where each chunk may have an overlapping segment with the next chunk. The overlapping segments might vary in wording but convey similar meaning. Each chunk is enclosed within <CHUNK_N_START> and <CHUNK_N_END> tags, where N is the chunk index starting from 0. |
| 57 | |
| 58 | **OUTPUT** |
| 59 | A single, consolidated text of the short story without unnatural repetitions or disruptions. The output should maintain the original narrative structure, tone, and details, with smooth transitions between originally adjacent chunks. |
| 60 | |
| 61 | **GUIDELINES** |
| 62 | 1. Analyze the input chunks sequentially. For each adjacent pair (e.g., Chunk N and Chunk N+1), compare the end of Chunk N and the beginning of Chunk N+1 to detect overlapping content. |
| 63 | 2. If the overlapping segments are semantically equivalent but phrased differently, merge them by retaining the most natural or contextually appropriate version (prioritize the version from the later chunk if both are equally valid, but avoid introducing inconsistency). |
| 64 | 3. If the overlapping segments are not perfectly equivalent (e.g., one contains additional details), integrate the meaningful information without duplication, ensuring no loss of content. |
| 65 | 4. Preserve all non-overlapping text exactly as it appears in the original chunks. Do not modify, paraphrase, or omit any unique content. |
| 66 | 5. Ensure the merged text is fluent and coherent, without abrupt jumps or redundant phrases. |
| 67 | 6. If no overlap is detected between two chunks, concatenate them directly without changes. |
| 68 | 7. Do not invent new content or alter the original narrative beyond handling the overlaps. |
| 69 | 8. The language of output should be consistent with the original text. |
| 70 | """ |
| 71 | |
| 72 | human_prompt_template_aggregate = \ |
| 73 | """ |
| 74 | {chunks} |
| 75 | """ |
| 76 | |
| 77 | |
| 78 | |
| 79 | |
| 80 | class NovelCompressor: |
| 81 | def __init__( |
| 82 | self, |
| 83 | api_key: str, |
| 84 | base_url: str, |
| 85 | chat_model: str, |
| 86 | chunk_size: int = 65536, |
| 87 | chunk_overlap: int = 8192, |
| 88 | ): |
| 89 | self.chat_model = init_chat_model( |
| 90 | model=chat_model, |
| 91 | api_key=api_key, |
| 92 | base_url=base_url, |
| 93 | model_provider="openai", |
| 94 | ) |
| 95 | |
| 96 | self.splitter = RecursiveCharacterTextSplitter( |
| 97 | chunk_size=chunk_size, |
| 98 | chunk_overlap=chunk_overlap, |
| 99 | ) |
| 100 | |
| 101 | |
| 102 | def split( |
| 103 | self, |
| 104 | novel_text: str, |
| 105 | ): |
| 106 | novel_chunks = self.splitter.split_text(novel_text) |
| 107 | return novel_chunks |
| 108 | |
| 109 | |
| 110 | async def compress( |
| 111 | self, |
| 112 | index_chunk_pairs: List[Tuple[int, str]], |
| 113 | max_concurrent_tasks: int = 5, |
| 114 | ) -> str: |
| 115 | sem = asyncio.Semaphore(max_concurrent_tasks) |
| 116 | |
| 117 | tasks = [ |
| 118 | self.compress_single_novel_chunk(sem, index, novel_chunk) |
| 119 | for index, novel_chunk in index_chunk_pairs |
| 120 | ] |
| 121 | compressed_novel_chunks = await asyncio.gather(*tasks) |
| 122 | return compressed_novel_chunks |
| 123 | |
| 124 | |
| 125 | async def compress_single_novel_chunk( |
| 126 | self, |
| 127 | semaphore: asyncio.Semaphore, |
| 128 | index, |
| 129 | novel_chunk: str, |
| 130 | ) -> str: |
| 131 | async with semaphore: |
| 132 | logging.info(f"Compressing novel chunk {index}") |
| 133 | messages = [ |
| 134 | SystemMessage( |
| 135 | content=system_prompt_template_compress_novel_chunk |
| 136 | ), |
| 137 | HumanMessage( |
| 138 | content=human_prompt_template_compress_novel_chunk.format( |
| 139 | novel_chunk=novel_chunk |
| 140 | ) |
| 141 | ), |
| 142 | ] |
| 143 | response = await self.chat_model.ainvoke(messages) |
| 144 | compressed_novel_chunk = response.content |
| 145 | logging.info(f"Compressed novel chunk {index}") |
| 146 | return index, compressed_novel_chunk |
| 147 | |
| 148 | |
| 149 | def aggregate( |
| 150 | self, |
| 151 | compressed_novel_chunks: List[str], |
| 152 | ): |
| 153 | chunks_str = "\n".join([ |
| 154 | f"<CHUNK_{i}_START>\n{chunk}\n<CHUNK_{i}_END>" |
| 155 | for i, chunk in enumerate(compressed_novel_chunks) |
| 156 | ]) |
| 157 | |
| 158 | messages = [ |
| 159 | SystemMessage( |
| 160 | content=system_prompt_template_aggregate |
| 161 | ), |
| 162 | HumanMessage( |
| 163 | content=human_prompt_template_aggregate.format( |
| 164 | chunks=chunks_str |
| 165 | ) |
| 166 | ), |
| 167 | ] |
| 168 | response = self.chat_model.invoke(messages) |
| 169 | aggregated_novel = response.content |
| 170 | return aggregated_novel |
| 171 | |
| 172 |