The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
SemanticChunker : Feature Addition ("Semantic Splitting with gradient") (#22895)
Raviraj858ce264efba
diff --git a/docs/docs/how_to/semantic-chunker.ipynb b/docs/docs/how_to/semantic-chunker.ipynbindex fbf50f3f57..ff952b2d81 100644--- a/docs/docs/how_to/semantic-chunker.ipynb+++ b/docs/docs/how_to/semantic-chunker.ipynb@@ -297,13 +297,67 @@"print(len(docs))"]},+ {+ "cell_type": "markdown",+ "source": [+ "### Gradient\n",+ "\n",+ "In this method, the gradient of distance is used to split chunks along with the percentile method.\n",+ "This method is useful when chunks are highly correlated with each other or specific to a domain e.g. legal or medical. The idea is to apply anomaly detection on gradient array so that the distribution become wider and easy to identify boundaries in highly semantic data."+ ],+ "metadata": {+ "collapsed": false+ },+ "id": "423c6e099e94ca69"+ },{"cell_type": "code","execution_count": null,"id": "b1f65472","metadata": {},"outputs": [],- "source": []+ "source": [+ "text_splitter = SemanticChunker(\n",+ " OpenAIEmbeddings(), breakpoint_threshold_type=\"gradient\"\n",+ ")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman.\n"+ ]+ }+ ],+ "source": [+ "docs = text_splitter.create_documents([state_of_the_union])\n",+ "print(docs[0].page_content)"+ ],+ "metadata": {},+ "id": "e9f393d316ce1f6c"+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "26\n"+ ]+ }+ ],+ "source": [+ "print(len(docs))"+ ],+ "metadata": {},+ "id": "a407cd57f02a0db4"}],"metadata": {diff --git a/libs/experimental/langchain_experimental/text_splitter.py b/libs/experimental/langchain_experimental/text_splitter.pyindex 1d0c462fa2..4ef5796f59 100644--- a/libs/experimental/langchain_experimental/text_splitter.py+++ b/libs/experimental/langchain_experimental/text_splitter.py@@ -84,11 +84,14 @@ def calculate_cosine_distances(sentences: List[dict]) -> Tuple[List[float], Listreturn distances, sentences-BreakpointThresholdType = Literal["percentile", "standard_deviation", "interquartile"]+BreakpointThresholdType = Literal[+ "percentile", "standard_deviation", "interquartile", "gradient"+]BREAKPOINT_DEFAULTS: Dict[BreakpointThresholdType, float] = {"percentile": 95,"standard_deviation": 3,"interquartile": 1.5,+ "gradient": 95,}@@ -127,23 +130,34 @@ class SemanticChunker(BaseDocumentTransformer):else:self.breakpoint_threshold_amount = breakpoint_threshold_amount- def _calculate_breakpoint_threshold(self, distances: List[float]) -> float:+ def _calculate_breakpoint_threshold(+ self, distances: List[float]+ ) -> Tuple[float, List[float]]:if self.breakpoint_threshold_type == "percentile":return cast(float,np.percentile(distances, self.breakpoint_threshold_amount),- )+ ), distanceselif self.breakpoint_threshold_type == "standard_deviation":return cast(float,np.mean(distances)+ self.breakpoint_threshold_amount * np.std(distances),- )+ ), distanceselif self.breakpoint_threshold_type == "interquartile":q1, q3 = np.percentile(distances, [25, 75])iqr = q3 - q1- return np.mean(distances) + self.breakpoint_threshold_amount * iqr+ return np.mean(+ distances+ ) + self.breakpoint_threshold_amount * iqr, distances+ elif self.breakpoint_threshold_type == "gradient":+ # Calculate the threshold based on the distribution of gradient of distance array. # noqa: E501+ distance_gradient = np.gradient(distances, range(0, len(distances)))+ return cast(+ float,+ np.percentile(distance_gradient, self.breakpoint_threshold_amount),+ ), distance_gradientelse:raise ValueError(f"Got unexpected `breakpoint_threshold_type`: "@@ -201,13 +215,17 @@ class SemanticChunker(BaseDocumentTransformer):distances, sentences = self._calculate_sentence_distances(single_sentences_list)if self.number_of_chunks is not None:breakpoint_distance_threshold = self._threshold_from_clusters(distances)+ breakpoint_array = distanceselse:- breakpoint_distance_threshold = self._calculate_breakpoint_threshold(- distances- )+ (+ breakpoint_distance_threshold,+ breakpoint_array,+ ) = self._calculate_breakpoint_threshold(distances)indices_above_thresh = [- i for i, x in enumerate(distances) if x > breakpoint_distance_threshold+ i+ for i, x in enumerate(breakpoint_array)+ if x > breakpoint_distance_threshold]chunks = []