💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add service task and tabular pipeline. Fix bug with spawning API. #130
davidmezzetti593c3fdd1ce0
diff --git a/examples/workflows.py b/examples/workflows.pyindex c26fd127..8c939df3 100644--- a/examples/workflows.py+++ b/examples/workflows.py@@ -21,8 +21,8 @@ import streamlit as stimport txtai.api.applicationfrom txtai.embeddings import Documents, Embeddings-from txtai.pipeline import Segmentation, Summary, Textractor, Transcription, Translation-from txtai.workflow import Workflow, Task, UrlTask+from txtai.pipeline import Segmentation, Summary, Tabular, Textractor, Transcription, Translation+from txtai.workflow import ServiceTask, Task, UrlTask, Workflowclass Server(uvicorn.Server):@@ -49,6 +49,7 @@ class Server(uvicorn.Server):Runs threaded server service."""+ # pylint: disable=W0201thread = threading.Thread(target=self.run)thread.start()try:@@ -57,6 +58,7 @@ class Server(uvicorn.Server):yieldfinally:+ self.should_exit = Truethread.join()@@ -98,6 +100,19 @@ class Application:value = st.sidebar.text_input(label)return int(value) if value else None+ def split(self, text):+ """+ Splits text on commas and returns a list.++ Args:+ text: input text++ Returns:+ list+ """++ return [x.strip() for x in text.split(",")]+def options(self, component):"""Extracts component settings into a component configuration dict.@@ -112,7 +127,14 @@ class Application:options = {"type": component}st.sidebar.markdown("---")- if component == "summary":++ if component == "embeddings":+ st.sidebar.markdown("**Embeddings Index** \n*Index workflow output*")+ options["index"] = st.sidebar.text_input("Embeddings storage path")+ options["path"] = st.sidebar.text_area("Embeddings model path", value="sentence-transformers/nli-mpnet-base-v2")+ options["upsert"] = st.sidebar.checkbox("Upsert")++ elif component == "summary":st.sidebar.markdown("**Summary** \n*Abstractive text summarization*")options["path"] = st.sidebar.text_input("Model", value="sshleifer/distilbart-cnn-12-6")options["minlength"] = self.number("Min length")@@ -130,6 +152,24 @@ class Application:options["join"] = st.sidebar.checkbox("Join tokenized")options["minlength"] = self.number("Min section length")+ elif component == "service":+ options["url"] = st.sidebar.text_input("URL")+ options["method"] = st.sidebar.selectbox("Method", ["get", "post"], index=0)+ options["params"] = st.sidebar.text_input("URL parameters")+ options["batch"] = st.sidebar.checkbox("Run as batch", value=True)+ options["extract"] = st.sidebar.text_input("Subsection(s) to extract")++ if options["params"]:+ options["params"] = {key: None for key in self.split(options["params"])}+ if options["extract"]:+ options["extract"] = self.split(options["extract"])++ elif component == "tabular":+ options["idcolumn"] = st.sidebar.text_input("Id columns")+ options["textcolumns"] = st.sidebar.text_input("Text columns")+ if options["textcolumns"]:+ options["textcolumns"] = self.split(options["textcolumns"])+elif component == "transcribe":st.sidebar.markdown("**Transcribe** \n*Transcribe audio to text*")options["path"] = st.sidebar.text_input("Model", value="facebook/wav2vec2-base-960h")@@ -138,12 +178,6 @@ class Application:st.sidebar.markdown("**Translate** \n*Machine translation*")options["target"] = st.sidebar.text_input("Target language code", value="en")- elif component == "embeddings":- st.sidebar.markdown("**Embeddings Index** \n*Index workflow output*")- options["index"] = st.sidebar.text_input("Embeddings storage path")- options["path"] = st.sidebar.text_area("Embeddings model path", value="sentence-transformers/nli-mpnet-base-v2")- options["upsert"] = st.sidebar.checkbox("Upsert")-return optionsdef build(self, components):@@ -164,14 +198,26 @@ class Application:wtype = component.pop("type")self.components[wtype] = component- if wtype == "summary":- self.pipelines[wtype] = Summary(component.pop("path"))- tasks.append(Task(lambda x: self.pipelines["summary"](x, **self.components["summary"])))+ if wtype == "embeddings":+ self.embeddings = Embeddings({**component})+ self.documents = Documents()+ tasks.append(Task(self.documents.add, unpack=False))elif wtype == "segment":self.pipelines[wtype] = Segmentation(**self.components["segment"])tasks.append(Task(self.pipelines["segment"]))+ elif wtype == "service":+ tasks.append(ServiceTask(**self.components["service"]))++ elif wtype == "summary":+ self.pipelines[wtype] = Summary(component.pop("path"))+ tasks.append(Task(lambda x: self.pipelines["summary"](x, **self.components["summary"])))++ elif wtype == "tabular":+ self.pipelines[wtype] = Tabular(**self.components["tabular"])+ tasks.append(Task(self.pipelines["tabular"]))+elif wtype == "textract":self.pipelines[wtype] = Textractor(**self.components["textract"])tasks.append(UrlTask(self.pipelines["textract"]))@@ -184,11 +230,6 @@ class Application:self.pipelines[wtype] = Translation()tasks.append(Task(lambda x: self.pipelines["translate"](x, **self.components["translate"])))- elif wtype == "embeddings":- self.embeddings = Embeddings({**component})- self.documents = Documents()- tasks.append(Task(self.documents.add, unpack=False))-self.workflow = Workflow(tasks)def yaml(self, components):@@ -219,6 +260,15 @@ class Application:data["segmentation"] = componenttasks.append({"action": "segmentation"})+ elif wtype == "service":+ config = dict(**component)+ config["task"] = "service"+ tasks.append(config)++ elif wtype == "tabular":+ data["tabular"] = component+ tasks.append({"action": "tabular"})+elif wtype == "textract":data["textractor"] = componenttasks.append({"action": "textractor", "task": "url"})@@ -275,6 +325,19 @@ class Application:uid += 1stop.empty()+ def find(self, key):+ """+ Lookup record from cached data by uid key.++ Args:+ key: uid to search for++ Returns:+ text for matching uid+ """++ return [text for uid, text, _ in self.data if uid == key][0]+def process(self, data):"""Processes the current application action.@@ -296,7 +359,7 @@ class Application:# Build embeddings indexif self.documents:# Cache data- self.data = [x[1] for x in self.documents]+ self.data = list(self.documents)with st.spinner("Building embedding index...."):self.embeddings.index(self.documents)@@ -326,9 +389,25 @@ class Application:)if query:- df = pd.DataFrame([{"content": self.data[uid], "score": score} for uid, score in self.embeddings.search(query, limit)])+ df = pd.DataFrame([{"content": self.find(uid), "score": score} for uid, score in self.embeddings.search(query, limit)])st.table(df)+ def parse(self, data):+ """+ Parse input data, splits on new lines depending on type of tasks and format of input.++ Args:+ data: input data++ Returns:+ parsed data+ """++ if re.match(r"^(http|https|file):\/\/", data) or (self.workflow and isinstance(self.workflow.tasks[0], ServiceTask)):+ return [x for x in data.split("\n") if x]++ return [data]+def run(self):"""Runs Streamlit application.@@ -338,7 +417,8 @@ class Application:st.sidebar.markdown("# Workflow builder \n*Build and apply workflows to data* ")# Get selected components- selected = st.sidebar.multiselect("Select components", ["embeddings", "segment", "summary", "textract", "transcribe", "translate"])+ components = ["embeddings", "segment", "service", "summary", "tabular", "textract", "transcribe", "translate"]+ selected = st.sidebar.multiselect("Select components", components)# Get selected optionscomponents = [self.options(component) for component in selected]@@ -367,7 +447,7 @@ class Application:data = st.text_area("Input", height=10)# Parse text items- data = [x for x in data.split("\n") if x] if re.match(r"^(http|https|file):\/\/", data) else [data]+ data = self.parse(data)# Process current actionself.process(data)