diff --git a/scrapegraphai/nodes/fetch_node.py b/scrapegraphai/nodes/fetch_node.py
index c55b96f64..68a4dea59 100644
--- a/scrapegraphai/nodes/fetch_node.py
+++ b/scrapegraphai/nodes/fetch_node.py
@@ -293,19 +293,26 @@ def handle_web_source(self, state, source):
if not self.cut:
parsed_content = cleanup_html(response, source)
+ else:
+ parsed_content = response.text
if (
isinstance(self.llm_model, (ChatOpenAI, AzureChatOpenAI))
and not self.script_creator
or (self.force and not self.script_creator)
):
- parsed_content = convert_to_md(source, parsed_content)
+ parsed_content = convert_to_md(parsed_content, source)
+ document = [
+ Document(page_content=response.text, metadata={"source": source})
+ ]
compressed_document = [Document(page_content=parsed_content)]
else:
self.logger.warning(
f"Failed to retrieve contents from the webpage at url: {source}"
)
+ document = [Document(page_content="", metadata={"source": source})]
+ compressed_document = document
else:
loader_kwargs = {}
@@ -395,7 +402,7 @@ def handle_web_source(self, state, source):
and not self.script_creator
and not self.openai_md_enabled
):
- parsed_content = convert_to_md(document[0].page_content, parsed_content)
+ parsed_content = convert_to_md(document[0].page_content, source)
compressed_document = [
Document(page_content=parsed_content, metadata={"source": "html file"})
diff --git a/tests/nodes/fetch_node_test.py b/tests/nodes/fetch_node_test.py
index 91144daab..29a20d90b 100644
--- a/tests/nodes/fetch_node_test.py
+++ b/tests/nodes/fetch_node_test.py
@@ -3,6 +3,54 @@
from scrapegraphai.nodes import FetchNode
+def test_fetch_html_convert_to_md_uses_source_as_baseurl(mocker):
+ """convert_to_md must receive the fetched page's URL as baseurl, not the HTML itself."""
+ content = "link"
+ mock_loader_cls = mocker.patch("scrapegraphai.nodes.fetch_node.ChromiumLoader")
+ mock_loader = mock_loader_cls.return_value
+ mock_loader.load.return_value = [Document(page_content=content)]
+ mock_convert = mocker.patch(
+ "scrapegraphai.nodes.fetch_node.convert_to_md", return_value="converted"
+ )
+
+ node = FetchNode(
+ input="url | local_dir",
+ output=["doc_content"],
+ node_config={"headless": False, "force": True},
+ )
+ source = "https://scrapegraph-ai.com/example"
+ result = node.execute({"url": source})
+
+ mock_convert.assert_called_once_with(content, source)
+ assert result["doc_content"][0].page_content == "converted"
+
+
+def test_fetch_html_use_soup_with_default_cut_does_not_raise(mocker):
+ """use_soup with the default cut=True must not raise UnboundLocalError and
+ must call convert_to_md with (html, source), not (source, html)."""
+ html = "link"
+ mock_response = mocker.Mock()
+ mock_response.status_code = 200
+ mock_response.text = html
+ mocker.patch(
+ "scrapegraphai.nodes.fetch_node.requests.get", return_value=mock_response
+ )
+ mock_convert = mocker.patch(
+ "scrapegraphai.nodes.fetch_node.convert_to_md", return_value="converted"
+ )
+
+ node = FetchNode(
+ input="url | local_dir",
+ output=["doc_content"],
+ node_config={"use_soup": True, "force": True},
+ )
+ source = "https://scrapegraph-ai.com/example"
+ result = node.execute({"url": source})
+
+ mock_convert.assert_called_once_with(html, source)
+ assert result["doc_content"][0].page_content == "converted"
+
+
def test_fetch_html(mocker):
title = "ScrapeGraph AI"
link_url = "https://github.com/VinciGit00/Scrapegraph-ai"