Skip to content

Commit 2dd7817

Browse files
PeriniMPeriniM
committed
feat: added verbose flag to suppress print statements
1 parent 5ac97e2 commit 2dd7817

20 files changed

+155
-58
lines changed

‎examples/groq/smart_scraper_groq.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,16 @@
1818

1919
graph_config = {
2020
"llm": {
21-
"api_key": groq_key,
2221
"model": "groq/gemma-7b-it",
22+
"api_key": groq_key,
23+
"temperature": 0
24+
},
25+
"embeddings": {
26+
"model": "ollama/nomic-embed-text",
27+
"temperature": 0,
28+
"base_url": "http://localhost:11434", # set ollama URL arbitrarily
2329
},
30+
"headless": False
2431
}
2532

2633
# ************************************************

‎examples/local_models/result.json

-1
This file was deleted.

‎examples/openai/result.json

-1
This file was deleted.

‎examples/openai/smart_scraper_openai.py

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"api_key": openai_key,
2222
"model": "gpt-3.5-turbo",
2323
},
24+
"verbose":False,
2425
}
2526

2627
# ************************************************

‎scrapegraphai/graphs/abstract_graph.py

+6
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ def __init__(self, prompt: str, config: dict, source: Optional[str] = None):
2222
self.llm_model = self._create_llm(config["llm"])
2323
self.embedder_model = self.llm_model if "embeddings" not in config else self._create_llm(
2424
config["embeddings"])
25+
26+
# Set common configuration parameters
27+
self.verbose = True if config is None else config.get("verbose", False)
28+
self.headless = True if config is None else config.get("headless", True)
29+
30+
# Create the graph
2531
self.graph = self._create_graph()
2632
self.final_state = None
2733
self.execution_info = None

‎scrapegraphai/graphs/base_graph.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import time
55
import warnings
66
from langchain_community.callbacks import get_openai_callback
7-
7+
from typing import Tuple
88

99
class BaseGraph:
1010
"""
@@ -56,7 +56,7 @@ def _create_edges(self, edges: list) -> dict:
5656
edge_dict[from_node.node_name] = to_node.node_name
5757
return edge_dict
5858

59-
def execute(self, initial_state: dict) -> (dict, list):
59+
def execute(self, initial_state: dict) -> Tuple[dict, list]:
6060
"""
6161
Executes the graph by traversing nodes starting from the entry point. The execution
6262
follows the edges based on the result of each node's execution and continues until

‎scrapegraphai/graphs/json_scraper_graph.py

+14-3
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,35 @@ def _create_graph(self):
3232
fetch_node = FetchNode(
3333
input="json_dir",
3434
output=["doc"],
35+
node_config={
36+
"headless": self.headless,
37+
"verbose": self.verbose
38+
}
3539
)
3640
parse_node = ParseNode(
3741
input="doc",
3842
output=["parsed_doc"],
39-
node_config={"chunk_size": self.model_token}
43+
node_config={
44+
"chunk_size": self.model_token,
45+
"verbose": self.verbose
46+
}
4047
)
4148
rag_node = RAGNode(
4249
input="user_prompt & (parsed_doc | doc)",
4350
output=["relevant_chunks"],
4451
node_config={
4552
"llm": self.llm_model,
46-
"embedder_model": self.embedder_model
53+
"embedder_model": self.embedder_model,
54+
"verbose": self.verbose
4755
}
4856
)
4957
generate_answer_node = GenerateAnswerNode(
5058
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
5159
output=["answer"],
52-
node_config={"llm": self.llm_model},
60+
node_config={
61+
"llm": self.llm_model,
62+
"verbose": self.verbose
63+
}
5364
)
5465

5566
return BaseGraph(

‎scrapegraphai/graphs/search_graph.py

+18-5
Original file line numberDiff line numberDiff line change
@@ -24,30 +24,43 @@ def _create_graph(self):
2424
search_internet_node = SearchInternetNode(
2525
input="user_prompt",
2626
output=["url"],
27-
node_config={"llm": self.llm_model}
27+
node_config={
28+
"llm": self.llm_model,
29+
"verbose": self.verbose
30+
}
2831
)
2932
fetch_node = FetchNode(
3033
input="url | local_dir",
3134
output=["doc"],
32-
node_config={"headless": True if self.config is None else self.config.get("headless", True)}
35+
node_config={
36+
"headless": self.headless,
37+
"verbose": self.verbose
38+
}
3339
)
3440
parse_node = ParseNode(
3541
input="doc",
3642
output=["parsed_doc"],
37-
node_config={"chunk_size": self.model_token}
43+
node_config={
44+
"chunk_size": self.model_token,
45+
"verbose": self.verbose
46+
}
3847
)
3948
rag_node = RAGNode(
4049
input="user_prompt & (parsed_doc | doc)",
4150
output=["relevant_chunks"],
4251
node_config={
4352
"llm": self.llm_model,
44-
"embedder_model": self.embedder_model
53+
"embedder_model": self.embedder_model,
54+
"verbose": self.verbose
4555
}
4656
)
4757
generate_answer_node = GenerateAnswerNode(
4858
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
4959
output=["answer"],
50-
node_config={"llm": self.llm_model},
60+
node_config={
61+
"llm": self.llm_model,
62+
"verbose": self.verbose
63+
}
5164
)
5265

5366
return BaseGraph(

‎scrapegraphai/graphs/smart_scraper_graph.py

+15-5
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def __init__(self, prompt: str, source: str, config: dict):
2424
super().__init__(prompt, config, source)
2525

2626
self.input_key = "url" if source.startswith("http") else "local_dir"
27-
27+
2828

2929
def _create_graph(self):
3030
"""
@@ -33,25 +33,35 @@ def _create_graph(self):
3333
fetch_node = FetchNode(
3434
input="url | local_dir",
3535
output=["doc"],
36-
node_config={"headless": True if self.config is None else self.config.get("headless", True)}
36+
node_config={
37+
"headless": self.headless,
38+
"verbose": self.verbose
39+
}
3740
)
3841
parse_node = ParseNode(
3942
input="doc",
4043
output=["parsed_doc"],
41-
node_config={"chunk_size": self.model_token}
44+
node_config={
45+
"chunk_size": self.model_token,
46+
"verbose": self.verbose
47+
}
4248
)
4349
rag_node = RAGNode(
4450
input="user_prompt & (parsed_doc | doc)",
4551
output=["relevant_chunks"],
4652
node_config={
4753
"llm": self.llm_model,
48-
"embedder_model": self.embedder_model
54+
"embedder_model": self.embedder_model,
55+
"verbose": self.verbose
4956
}
5057
)
5158
generate_answer_node = GenerateAnswerNode(
5259
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
5360
output=["answer"],
54-
node_config={"llm": self.llm_model},
61+
node_config={
62+
"llm": self.llm_model,
63+
"verbose": self.verbose
64+
}
5565
)
5666

5767
return BaseGraph(

‎scrapegraphai/graphs/speech_graph.py

+18-6
Original file line numberDiff line numberDiff line change
@@ -35,31 +35,43 @@ def _create_graph(self):
3535
fetch_node = FetchNode(
3636
input="url | local_dir",
3737
output=["doc"],
38-
node_config={"headless": True if self.config is None else self.config.get("headless", True)}
38+
node_config={
39+
"headless": self.headless,
40+
"verbose": self.verbose
41+
}
3942
)
4043
parse_node = ParseNode(
4144
input="doc",
4245
output=["parsed_doc"],
43-
node_config={"chunk_size": self.model_token}
46+
node_config={
47+
"chunk_size": self.model_token,
48+
"verbose": self.verbose
49+
}
4450
)
4551
rag_node = RAGNode(
4652
input="user_prompt & (parsed_doc | doc)",
4753
output=["relevant_chunks"],
4854
node_config={
4955
"llm": self.llm_model,
50-
"embedder_model": self.embedder_model
56+
"embedder_model": self.embedder_model,
57+
"verbose": self.verbose
5158
}
5259
)
5360
generate_answer_node = GenerateAnswerNode(
5461
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
5562
output=["answer"],
56-
node_config={"llm": self.llm_model},
63+
node_config={
64+
"llm": self.llm_model,
65+
"verbose": self.verbose
66+
}
5767
)
5868
text_to_speech_node = TextToSpeechNode(
5969
input="answer",
6070
output=["audio"],
61-
node_config={"tts_model": OpenAITextToSpeech(
62-
self.config["tts_model"])},
71+
node_config={
72+
"tts_model": OpenAITextToSpeech(self.config["tts_model"]),
73+
"verbose": self.verbose
74+
}
6375
)
6476

6577
return BaseGraph(

‎scrapegraphai/graphs/xml_scraper_graph.py

+14-3
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,35 @@ def _create_graph(self):
3232
fetch_node = FetchNode(
3333
input="xml_dir",
3434
output=["doc"],
35+
node_config={
36+
"headless": self.headless,
37+
"verbose": self.verbose
38+
}
3539
)
3640
parse_node = ParseNode(
3741
input="doc",
3842
output=["parsed_doc"],
39-
node_config={"chunk_size": self.model_token}
43+
node_config={
44+
"chunk_size": self.model_token,
45+
"verbose": self.verbose
46+
}
4047
)
4148
rag_node = RAGNode(
4249
input="user_prompt & (parsed_doc | doc)",
4350
output=["relevant_chunks"],
4451
node_config={
4552
"llm": self.llm_model,
46-
"embedder_model": self.embedder_model
53+
"embedder_model": self.embedder_model,
54+
"verbose": self.verbose
4755
}
4856
)
4957
generate_answer_node = GenerateAnswerNode(
5058
input="user_prompt & (relevant_chunks | parsed_doc | doc)",
5159
output=["answer"],
52-
node_config={"llm": self.llm_model},
60+
node_config={
61+
"llm": self.llm_model,
62+
"verbose": self.verbose
63+
}
5364
)
5465

5566
return BaseGraph(

‎scrapegraphai/nodes/fetch_node.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def __init__(self, input: str, output: List[str], node_config: Optional[dict], n
4747
super().__init__(node_name, "node", input, output, 1)
4848

4949
self.headless = True if node_config is None else node_config.get("headless", True)
50+
self.verbose = True if node_config is None else node_config.get("verbose", False)
5051

5152
def execute(self, state):
5253
"""
@@ -63,7 +64,8 @@ def execute(self, state):
6364
KeyError: If the 'url' key is not found in the state, indicating that the
6465
necessary information to perform the operation is missing.
6566
"""
66-
print(f"--- Executing {self.node_name} Node ---")
67+
if self.verbose:
68+
print(f"--- Executing {self.node_name} Node ---")
6769

6870
# Interpret input keys based on the provided input expression
6971
input_keys = self.get_input_keys(state)

‎scrapegraphai/nodes/generate_answer_node.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ def __init__(self, input: str, output: List[str], node_config: dict,
4949
"""
5050
super().__init__(node_name, "node", input, output, 2, node_config)
5151
self.llm_model = node_config["llm"]
52+
self.verbose = True if node_config is None else node_config.get("verbose", False)
5253

5354
def execute(self, state):
5455
"""
@@ -69,7 +70,8 @@ def execute(self, state):
6970
that the necessary information for generating an answer is missing.
7071
"""
7172

72-
print(f"--- Executing {self.node_name} Node ---")
73+
if self.verbose:
74+
print(f"--- Executing {self.node_name} Node ---")
7375

7476
# Interpret input keys based on the provided input expression
7577
input_keys = self.get_input_keys(state)
@@ -116,7 +118,7 @@ def execute(self, state):
116118
chains_dict = {}
117119

118120
# Use tqdm to add progress bar
119-
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks")):
121+
for i, chunk in enumerate(tqdm(doc, desc="Processing chunks", disable=not self.verbose)):
120122
if len(doc) == 1:
121123
prompt = PromptTemplate(
122124
template=template_no_chunks,

‎scrapegraphai/nodes/image_to_text_node.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ def __init__(self, input: str, output: List[str], node_config: dict,
2929
"""
3030
super().__init__(node_name, "node", input, output, 1, node_config)
3131
self.llm_model = node_config["llm_model"]
32+
self.verbose = True if node_config is None else node_config.get("verbose", False)
3233

3334
def execute(self, state: dict) -> dict:
3435
"""
@@ -40,9 +41,11 @@ def execute(self, state: dict) -> dict:
4041
Returns:
4142
dict: The updated state after executing this node.
4243
"""
43-
print("---GENERATING TEXT FROM IMAGE---")
44-
input_keys = self.get_input_keys(state)
4544

45+
if self.verbose:
46+
print("---GENERATING TEXT FROM IMAGE---")
47+
48+
input_keys = self.get_input_keys(state)
4649
input_data = [state[key] for key in input_keys]
4750
url = input_data[0]
4851

‎scrapegraphai/nodes/parse_node.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ def __init__(self, input: str, output: List[str], node_config: dict, node_name:
4040
"""
4141
super().__init__(node_name, "node", input, output, 1, node_config)
4242

43+
self.verbose = True if node_config is None else node_config.get("verbose", False)
44+
4345
def execute(self, state):
4446
"""
4547
Executes the node's logic to parse the HTML document based on specified tags.
@@ -60,7 +62,8 @@ def execute(self, state):
6062
information for parsing is missing.
6163
"""
6264

63-
print(f"--- Executing {self.node_name} Node ---")
65+
if self.verbose:
66+
print(f"--- Executing {self.node_name} Node ---")
6467

6568
# Interpret input keys based on the provided input expression
6669
input_keys = self.get_input_keys(state)

0 commit comments

Comments
 (0)