greydgl--pentestgpt
gelei
abe3be01b4
Major rewrite of PentestGPT to use an agentic pipeline architecture: Core Changes: - New event-driven architecture with EventBus for TUI-agent decoupling - Implemented AgentController with 5-state lifecycle (IDLE->RUNNING->PAUSED->COMPLETED->ERROR) - Added AgentBackend interface with ClaudeCodeBackend implementation - Session management with file-based persistence for resumable pentests - Langfuse integration for observability and tracing Interface: - New Textual-based TUI with real-time activity feed - Keyboard shortcuts: F1 help, Ctrl+P pause, Ctrl+Q quit - Enhanced CLI with --target, --instruction, --non-interactive, --debug flags Project Structure: - Moved legacy multi-LLM version (v0.15) to legacy/ directory - New pentestgpt/core/ for agent, controller, events, session modules - New pentestgpt/interface/ for TUI and CLI components - New pentestgpt/benchmark/ for xbow benchmark integration - Comprehensive test suite in tests/ with unit and integration tests DevOps: - Docker support with Ubuntu 24.04 container - GitHub Actions CI/CD pipeline - Makefile with dev commands (test, lint, format, typecheck) - Added xbow-validation-benchmarks as submodule
57 行
2.0 KiB
Python
57 行
2.0 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
|
|
|
|
def crawl_dotCMS_description_page(
|
|
url="https://www.dotcms.com/docs/latest/container-api", output_dir="outputs"
|
|
):
|
|
page = requests.get(url)
|
|
soup = BeautifulSoup(page.content, "html.parser")
|
|
|
|
# Extract the title of the page
|
|
title = soup.find("h1").get_text()
|
|
|
|
# Extract the subtitles and their descriptions and code chunks
|
|
subtitles = soup.find_all("h2")
|
|
parsed_subtitles = []
|
|
for subtitle in subtitles:
|
|
subtitle_title = subtitle.get_text()
|
|
subtitle_contents = subtitle.find_next_siblings(["p", "pre"])
|
|
subtitle_parsed_contents = []
|
|
description = ""
|
|
for content in subtitle_contents:
|
|
# Check if the content is a code block
|
|
if content.name == "pre" and content.code:
|
|
code = content.get_text()
|
|
# Add the previous description and code chunk to the list
|
|
if len(description) != 0: # If there is no description, don't add it
|
|
parsed_description = description.strip().replace("\n", " ")
|
|
parsed_code = code.strip().replace("\n", " ")
|
|
subtitle_parsed_contents.append([parsed_description, parsed_code])
|
|
|
|
else:
|
|
# Concatenate the non-code content into a single description string
|
|
description += (
|
|
"\n" + content.get_text() if description else content.get_text()
|
|
)
|
|
parsed_subtitles.append([subtitle_title, subtitle_parsed_contents])
|
|
|
|
# Save the results as a structured JSON object
|
|
title = title.strip().replace(" ", "_").lower()
|
|
output = {"title": title}
|
|
for parsed_subtitle in parsed_subtitles:
|
|
output[parsed_subtitle[0]] = parsed_subtitle[1]
|
|
|
|
with open(f"{output_dir}/{title}.json", "w") as f:
|
|
json.dump(output, f)
|
|
return output
|
|
|
|
|
|
def crawl_strapi_documentation(url, output_dir="outputs"):
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
output_dir = "outputs"
|