add examples for document_scraper

Author: VinciGit00

examples/anthropic/depth_search_graph_anthropic.py

@@ -10,7 +10,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "openai/gpt-4o-mini",
+        "model": "anthropic/claude-3-haiku-20240307",
     },
     "verbose": True,
     "headless": False,

examples/anthropic/document_scraper_anthropic.py

@@ -0,0 +1,42 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    }
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/anthropic/json_scraper_anthropic.py

@@ -4,7 +4,6 @@
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
 
 load_dotenv()
 
@@ -27,7 +26,7 @@
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
         "model": "anthropic/claude-3-haiku-20240307",
-    },
+    }
 }
 
 # ************************************************
@@ -42,15 +41,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")
-

examples/azure/depth_search_graph_azure.py

@@ -1,14 +1,12 @@
 """
-depth_search_graph_opeani example
+depth_search_graph_azure example
 """
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import DepthSearchGraph
 
 load_dotenv()
 
-openai_key = os.getenv("OPENAI_APIKEY")
-
 graph_config = {
    "llm": {
         "api_key": os.environ["AZURE_OPENAI_KEY"],

examples/azure/document_scraper_azure.py

@@ -0,0 +1,44 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.environ["AZURE_OPENAI_KEY"],
+        "model": "azure_openai/gpt-4o"
+    },
+    "verbose": True,
+    "headless": False
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/bedrock/csv_scraper_bedrock.py

@@ -10,7 +10,7 @@
 import pandas as pd
 
 from scrapegraphai.graphs import CSVScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 
@@ -48,13 +48,3 @@
 result = csv_scraper_graph.run()
 print(json.dumps(result, indent=4))
 
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = csv_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/bedrock/document_scraper_bedrock.py

@@ -0,0 +1,42 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+graph_config = {
+        "llm": {
+        "client": "client_name",
+        "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+        "temperature": 0.0
+    }
+}
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/deepseek/document_scraper_deepseek.py

@@ -0,0 +1,44 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek/deepseek-chat",
+        "api_key": deepseek_key,
+    },
+    "verbose": True,
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/deepseek/json_scraper_deepseek.py

@@ -4,7 +4,7 @@
 import os
 from dotenv import load_dotenv
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import prettify_exec_info
 
 load_dotenv()
 
@@ -44,14 +44,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/ernie/document_scraper_anthropic_ernie.py

@@ -0,0 +1,39 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from scrapegraphai.graphs import DocumentScraperGraph
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+graph_config = {
+    "llm": {
+        "model": "ernie/ernie-bot-turbo",
+        "ernie_client_id": "<ernie_client_id>",
+        "ernie_client_secret": "<ernie_client_secret>",
+        "temperature": 0.1
+    }
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))

examples/ernie/json_scraper_ernie.py

@@ -3,7 +3,7 @@
 """
 import os
 from scrapegraphai.graphs import JSONScraperGraph
-from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+from scrapegraphai.utils import  prettify_exec_info
 
 # ************************************************
 # Read the JSON file
@@ -41,14 +41,3 @@
 
 result = json_scraper_graph.run()
 print(result)
-
-# ************************************************
-# Get graph execution info
-# ************************************************
-
-graph_exec_info = json_scraper_graph.get_execution_info()
-print(prettify_exec_info(graph_exec_info))
-
-# Save to json or csv
-convert_to_csv(result, "result")
-convert_to_json(result, "result")

examples/fireworks/document_scraper_anthropic_fireworks.py

@@ -0,0 +1,44 @@
+"""
+document_scraper example
+"""
+import os
+import json
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DocumentScraperGraph
+
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+fireworks_api_key = os.getenv("FIREWORKS_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": fireworks_api_key,
+        "model": "fireworks/accounts/fireworks/models/mixtral-8x7b-instruct"
+    },
+    "verbose": True,
+    "headless": False,
+}
+
+
+source = """
+    The Divine Comedy, Italian La Divina Commedia, original name La commedia, long narrative poem written in Italian 
+    circa 1308/21 by Dante. It is usually held to be one of the world s great works of literature. 
+    Divided into three major sections—Inferno, Purgatorio, and Paradiso—the narrative traces the journey of Dante 
+    from darkness and error to the revelation of the divine light, culminating in the Beatific Vision of God. 
+    Dante is guided by the Roman poet Virgil, who represents the epitome of human knowledge, from the dark wood 
+    through the descending circles of the pit of Hell (Inferno). He then climbs the mountain of Purgatory, guided 
+    by the Roman poet Statius, who represents the fulfilment of human knowledge, and is finally led by his lifelong love, 
+    the Beatrice of his earlier poetry, through the celestial spheres of Paradise.
+"""
+
+pdf_scraper_graph = DocumentScraperGraph(
+    prompt="Summarize the text and find the main topics",
+    source=source,
+    config=graph_config,
+)
+result = pdf_scraper_graph.run()
+
+print(json.dumps(result, indent=4))