allignment

CHANGELOG.md

@@ -1,3 +1,22 @@
+## [1.31.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.31.0...v1.31.1) (2024-11-22)
+
+
+### Bug Fixes
+
+* add new model istance ([2f3cafe](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2f3cafeab0bce38571fa10d71f454b2a31766ddc))
+* fetch node regex ([e2af232](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e2af2326f6c56e2abcc7dd5de9acdfb710507e0a))
+* generate answer node timeout ([32ef554](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/32ef5547f1d864c750cd47c115be6f38a1931d2c))
+* timeout ([c243106](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c243106552cec3b1df254c0d0a45401eb2f5c89d))
+
+
+### CI
+
+* **release:** 1.31.0-beta.1 [skip ci] ([1df7eb0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/1df7eb0bcd923bc62fd19dddc0ce9b757e9742cf)), closes [#805](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/805) [#805](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/805)
+* **release:** 1.31.1-beta.1 [skip ci] ([86bf4f2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/86bf4f24021d6e73378495d5b2b3acbfa2ff8ed5)), closes [#805](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/805) [#805](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/805)
+* **release:** 1.31.1-beta.2 [skip ci] ([f247844](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f247844d81e018c749c3a9a7170ed3ceded5d483))
+* **release:** 1.31.1-beta.3 [skip ci] ([30b0156](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/30b0156d17aa23e99d203eb6c7dd4f42e1e83566))
+* **release:** 1.31.1-beta.4 [skip ci] ([b2720a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/b2720a452f023999e3b394636773b794941cc6a1))
+
 ## [1.31.1-beta.4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.31.1-beta.3...v1.31.1-beta.4) (2024-11-21)
 
 

Dockerfile

@@ -1,9 +1,9 @@
 FROM python:3.11-slim
 
-RUN apt-get update && apt-get upgrade -y
+RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
 
-RUN pip install scrapegraphai
-RUN pip install scrapegraphai[burr]
+RUN pip install --no-cache-dir scrapegraphai
+RUN pip install --no-cache-dir scrapegraphai[burr]
 
 RUN python3 -m playwright install-deps
 RUN python3 -m playwright install

examples/anthropic/depth_search_graph_anthropic.py

@@ -10,7 +10,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "openai/gpt-4o-mini",
+        "model": "anthropic/claude-3-haiku-20240307",
     },
     "verbose": True,
     "headless": False,

examples/anthropic/document_scraper_anthropic.py

@@ -0,0 +1,42 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    }
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/anthropic/json_scraper_anthropic.py

@@ -4,7 +4,6 @@
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
 
 load_dotenv()
 
@@ -27,7 +26,7 @@
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
         "model": "anthropic/claude-3-haiku-20240307",
-    },
+    }
 }
 
 # ************************************************
@@ -42,15 +41,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")
-

examples/azure/depth_search_graph_azure.py

@@ -1,14 +1,12 @@
 """
-depth_search_graph_opeani example
+depth_search_graph_azure example
 """
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import DepthSearchGraph
 
 load_dotenv()
 
-openai_key = os.getenv("OPENAI_APIKEY")
-
 graph_config = {
    "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],

examples/azure/document_scraper_azure.py

@@ -0,0 +1,44 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-4o"
+    },
+    "verbose": True,
+    "headless": False
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/bedrock/csv_scraper_bedrock.py

@@ -10,7 +10,7 @@
 import pandas as pd
 
 from scrapegraphai.graphs import CSVScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 
@@ -48,13 +48,3 @@
 result = csv_scraper_graph.run()
 print(json.dumps(result, indent=4))
 
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = csv_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/bedrock/document_scraper_bedrock.py

@@ -0,0 +1,42 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+        "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    }
+}
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/deepseek/document_scraper_deepseek.py

@@ -0,0 +1,44 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek/deepseek-chat",
+        "api_key": deepseek_key,
+    },
+    "verbose": True,
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/deepseek/json_scraper_deepseek.py

@@ -4,7 +4,7 @@
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 
@@ -44,14 +44,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/ernie/document_scraper_anthropic_ernie.py

@@ -0,0 +1,39 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from scrapegraphai.graphs import DocumentScraperGraph
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+graph_config = {
+    "llm": {
+        "model": "ernie/ernie-bot-turbo",
+        "ernie_client_id": "<ernie_client_id>",
+        "ernie_client_secret": "<ernie_client_secret>",
+        "temperature": 0.1
+    }
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/ernie/json_scraper_ernie.py

@@ -3,7 +3,7 @@
 """
 import os
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import  prettify_exec_info
 
 # ************************************************
 # Read the JSON file
@@ -41,14 +41,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")