Skip to content

fix(fetch): pass the correct baseurl to convert_to_md, fix undefined document in use_soup path #1151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Kumaresan-AI-Engineer wants to merge 1 commit into
base: pre/beta
Choose a base branch
from
+57 −2
Open

fix(fetch): pass the correct baseurl to convert_to_md, fix undefined document in use_soup path #1151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions scrapegraphai/nodes/fetch_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,19 +293,26 @@ def handle_web_source(self, state, source):

if not self.cut:
parsed_content = cleanup_html(response, source)
else:
parsed_content = response.text

if (
isinstance(self.llm_model, (ChatOpenAI, AzureChatOpenAI))
and not self.script_creator
or (self.force and not self.script_creator)
):
parsed_content = convert_to_md(source, parsed_content)
parsed_content = convert_to_md(parsed_content, source)

document = [
Document(page_content=response.text, metadata={"source": source})
]
compressed_document = [Document(page_content=parsed_content)]
else:
self.logger.warning(
f"Failed to retrieve contents from the webpage at url: {source}"
)
document = [Document(page_content="", metadata={"source": source})]
compressed_document = document
else:
loader_kwargs = {}

Expand Down Expand Up @@ -395,7 +402,7 @@ def handle_web_source(self, state, source):
and not self.script_creator
and not self.openai_md_enabled
):
parsed_content = convert_to_md(document[0].page_content, parsed_content)
parsed_content = convert_to_md(document[0].page_content, source)

compressed_document = [
Document(page_content=parsed_content, metadata={"source": "html file"})
Expand Down
48 changes: 48 additions & 0 deletions tests/nodes/fetch_node_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,54 @@
from scrapegraphai.nodes import FetchNode


def test_fetch_html_convert_to_md_uses_source_as_baseurl(mocker):
"""convert_to_md must receive the fetched page's URL as baseurl, not the HTML itself."""
content = "<html><body><a href='/relative'>link</a></body></html>"
mock_loader_cls = mocker.patch("scrapegraphai.nodes.fetch_node.ChromiumLoader")
mock_loader = mock_loader_cls.return_value
mock_loader.load.return_value = [Document(page_content=content)]
mock_convert = mocker.patch(
"scrapegraphai.nodes.fetch_node.convert_to_md", return_value="converted"
)

node = FetchNode(
input="url | local_dir",
output=["doc_content"],
node_config={"headless": False, "force": True},
)
source = "https://scrapegraph-ai.com/example"
result = node.execute({"url": source})

mock_convert.assert_called_once_with(content, source)
assert result["doc_content"][0].page_content == "converted"


def test_fetch_html_use_soup_with_default_cut_does_not_raise(mocker):
"""use_soup with the default cut=True must not raise UnboundLocalError and
must call convert_to_md with (html, source), not (source, html)."""
html = "<html><body><a href='/relative'>link</a></body></html>"
mock_response = mocker.Mock()
mock_response.status_code = 200
mock_response.text = html
mocker.patch(
"scrapegraphai.nodes.fetch_node.requests.get", return_value=mock_response
)
mock_convert = mocker.patch(
"scrapegraphai.nodes.fetch_node.convert_to_md", return_value="converted"
)

node = FetchNode(
input="url | local_dir",
output=["doc_content"],
node_config={"use_soup": True, "force": True},
)
source = "https://scrapegraph-ai.com/example"
result = node.execute({"url": source})

mock_convert.assert_called_once_with(html, source)
assert result["doc_content"][0].page_content == "converted"


def test_fetch_html(mocker):
title = "ScrapeGraph AI"
link_url = "https://github.com/VinciGit00/Scrapegraph-ai"
Expand Down