Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

4.7 KB
Raw Blame
import mimetypes
import re
from dataclasses import dataclass
from typing import Any
from urllib.parse import unquote

import charset_normalizer
import cloudscraper
from readabilipy import simple_json_from_html_string

from core.file import remote_fetcher
from core.rag.extractor import extract_processor
from core.rag.extractor.extract_processor import ExtractProcessor

FULL_TEMPLATE = """
TITLE: {title}
AUTHOR: {author}
TEXT:

{text}
"""


def page_result(text: str, cursor: int, max_length: int) -> str:
    """Page through `text` and return a substring of `max_length` characters starting from `cursor`."""
    return text[cursor : cursor + max_length]


def get_url(url: str, user_agent: str | None = None) -> str:
    """Fetch URL and return the contents as a string."""
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
        " Chrome/91.0.4472.124 Safari/537.36"
    }
    if user_agent:
        headers["User-Agent"] = user_agent

    main_content_type = None
    supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"]
    response = remote_fetcher.make_request("HEAD", url, headers=headers, follow_redirects=True, timeout=(5, 10))

    if response.status_code == 200:
        # check content-type
        content_type = response.headers.get("Content-Type")
        if content_type:
            main_content_type = response.headers.get("Content-Type").split(";")[0].strip()
        else:
            content_disposition = response.headers.get("Content-Disposition", "")
            filename_match = re.search(r'filename="([^"]+)"', content_disposition)
            if filename_match:
                filename = unquote(filename_match.group(1))
                extension = re.search(r"\.(\w+)$", filename)
                if extension:
                    main_content_type = mimetypes.guess_type(filename)[0]

        if main_content_type not in supported_content_types:
            return f"Unsupported content-type [{main_content_type}] of URL."

        if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES:
            return ExtractProcessor.load_from_url(url, return_text=True)

        response = remote_fetcher.make_request("GET", url, headers=headers, follow_redirects=True, timeout=(120, 300))
    elif response.status_code == 403:
        scraper = cloudscraper.create_scraper()
        object.__setattr__(scraper, "perform_request", remote_fetcher.make_request)
        response = scraper.get(url, headers=headers, timeout=(120, 300))

    if response.status_code != 200:
        return f"URL returned status code {response.status_code}."

    # Detect encoding using charset_normalizer
    detected_encoding = charset_normalizer.from_bytes(response.content).best()
    encoding = detected_encoding.encoding if detected_encoding else "utf-8"
    if encoding:
        try:
            content = response.content.decode(encoding)
        except (UnicodeDecodeError, TypeError):
            content = response.text
    else:
        content = response.text

    article = extract_using_readabilipy(content)

    if not article.text:
        return ""

    res = FULL_TEMPLATE.format(
        title=article.title,
        author=article.author,
        text=article.text,
    )

    return res


@dataclass
class Article:
    title: str
    author: str
    text: str


def extract_using_readabilipy(html: str):
    json_article: dict[str, Any] = simple_json_from_html_string(html, use_readability=False)
    # readabilipy returns plain_text as a list of dicts whose "text" values
    # contain the (possibly HTML-tagged) paragraph content; flatten them into
    # a single clean text block.
    plain_text = json_article.get("plain_text") or []
    text = "\n".join(
        stripped
        for item in plain_text
        if isinstance(item, dict) and (stripped := re.sub(r"<[^>]+>", "", str(item.get("text") or "")).strip())
    )
    article = Article(
        title=json_article.get("title") or "",
        author=json_article.get("byline") or "",
        text=text,
    )

    return article


def get_image_upload_file_ids(content):
    pattern = r"!\[image\]\((https?://.*?(file-preview|image-preview))\)"
    matches = re.findall(pattern, content)
    image_upload_file_ids = []
    for match in matches:
        if match[1] == "file-preview":
            content_pattern = r"files/([^/]+)/file-preview"
        else:
            content_pattern = r"files/([^/]+)/image-preview"
        content_match = re.search(content_pattern, match[0])
        if content_match:
            image_upload_file_id = content_match.group(1)
            image_upload_file_ids.append(image_upload_file_id)
    return image_upload_file_ids