Skip to content

Update Web Spider Param #364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
payoub125 merged 2 commits into from
Apr 3, 2026
Merged

Update Web Spider Param #364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3791b23
update web spider param
payoub125 Apr 3, 2026
88f873f
fix cursor comments
payoub125 Apr 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmd/discover.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,7 @@ func (a *WebScan) InitDiscoverCommand() {
}

// Get Config flags
ignoreBaseURLMatch, err := cmd.Flags().GetBool("ignore-base-url-match")
ignoreCrossDomain, err := cmd.Flags().GetBool("ignore-cross-domain")
if err != nil {
a.OutputSignal.AddError(err)
return
Expand Down Expand Up @@ -559,7 +559,7 @@ func (a *WebScan) InitDiscoverCommand() {
}

// Set Config
config := getDiscoverRouteConfig(target, ignoreBaseURLMatch, collectStaticAssets, spiderDepth, maxRedirects, verifyTLS, timeout, ignoreCrossDomainRedirects, threads, requestMethodConfig.RequestMethodEnum, requestMethodConfig.HeadlessConfig, requestMethodConfig.BrowserbaseConfig)
config := getDiscoverRouteConfig(target, ignoreCrossDomain, collectStaticAssets, spiderDepth, maxRedirects, verifyTLS, timeout, ignoreCrossDomainRedirects, threads, requestMethodConfig.RequestMethodEnum, requestMethodConfig.HeadlessConfig, requestMethodConfig.BrowserbaseConfig)

// Generate a report
report := discoverroute.PerformRouteCapture(cmd.Context(), config, requestMethodConfig.BrowserbaseSecrets)
Expand All @@ -569,7 +569,7 @@ func (a *WebScan) InitDiscoverCommand() {
// Target Flags
discoverRouteCmd.Flags().String("target", "", "URL target to discover routes from")
// Config Flags
discoverRouteCmd.Flags().Bool("ignore-base-url-match", false, "Add route even if it does not share the target's base URL")
discoverRouteCmd.Flags().Bool("ignore-cross-domain", true, "Ignore routes that do not share the target's base URL")
discoverRouteCmd.Flags().Bool("collect-static-assets", false, "Collect static assets from route discovery")
discoverRouteCmd.Flags().Int("spider-depth", 3, "Maximum depth for route spidering")
discoverRouteCmd.Flags().Int("max-redirects", 100, "Maximum number of redirects to follow")
Expand Down Expand Up @@ -808,11 +808,11 @@ func getDiscoverProbeConfig(targets []string, protocol string, maxRedirects int,
}

// getDiscoverRouteConfig builds the config for route discovery.
func getDiscoverRouteConfig(target string, ignoreBaseURLMatch bool, collectStaticAssets bool, spiderDepth int, maxRedirects int, verifyTLS bool, timeout int, ignoreCrossDomainRedirects bool, threads int, requestMethod common.RequestMethod, headlessConfig *common.HeadlessRequestConfig, browserbaseConfig *common.BrowserbaseRequestConfig) discover.DiscoverRouteConfig {
func getDiscoverRouteConfig(target string, ignoreCrossDomain bool, collectStaticAssets bool, spiderDepth int, maxRedirects int, verifyTLS bool, timeout int, ignoreCrossDomainRedirects bool, threads int, requestMethod common.RequestMethod, headlessConfig *common.HeadlessRequestConfig, browserbaseConfig *common.BrowserbaseRequestConfig) discover.DiscoverRouteConfig {
config := discover.DiscoverRouteConfig{
Target: target,
CollectStaticAssets: collectStaticAssets,
IgnoreBaseUrlMatch: ignoreBaseURLMatch,
IgnoreCrossDomain: ignoreCrossDomain,
SpiderDepth: spiderDepth,
MaxRedirects: maxRedirects,
VerifyTls: verifyTLS,
Expand Down
2 changes: 1 addition & 1 deletion cmd/pentest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -823,7 +823,7 @@ func getPentestRouteStaticAssetTakeoverConfig(target string, fingerprintFilePath
routeCaptureConfig := &discover.DiscoverRouteConfig{
Target: target,
CollectStaticAssets: true,
IgnoreBaseUrlMatch: true,
IgnoreCrossDomain: false,
SpiderDepth: 1,
VerifyTls: verifyTLS,
Timeout: max(timeout, 0),
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/discover.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ Flags:
--collect-static-assets Collect static assets from route discovery
--headless-path string Path to headless browser executable
-h, --help help for route
--ignore-base-url-match Add route even if it does not share the target's base URL
--ignore-cross-domain Ignore routes that do not share the target's base URL (default true)
--max-redirects int Maximum number of redirects to follow (default 100)
--min-dom-stabalize-time int Minimum time to wait for DOM stabilization in seconds (default 20)
--request-method string Request method to use (standard, headless, browserbase) (default "STANDARD")
Expand Down
2 changes: 1 addition & 1 deletion fern/definition/discover/route.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ types:
properties:
target: string
collectStaticAssets: boolean
ignoreBaseUrlMatch: boolean
ignoreCrossDomain: boolean
spiderDepth: integer
maxRedirects: integer
verifyTls: boolean
Expand Down
1 change: 0 additions & 1 deletion internal/discover/page/helpers/sensitivecontent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

// Internal
"github.com/Method-Security/webscan/configs"

// Generated
"github.com/Method-Security/webscan/generated/go/discover"
// External
Expand Down
1 change: 1 addition & 0 deletions internal/discover/page/page.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
// Standard
"context"
"fmt"

// Generated
common "github.com/Method-Security/webscan/generated/go/common"
"github.com/Method-Security/webscan/generated/go/discover"
Expand Down
6 changes: 3 additions & 3 deletions internal/discover/route/helpers/extractors/htmlExtractors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func ExtractFormRoutes(doc *goquery.Document, baseURL string, routeCaptureConfig
fullURL := discoverroutehelpers.ResolveURL(baseURL, action)

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
return
}

Expand Down Expand Up @@ -134,7 +134,7 @@ func ExtractAnchorRoutes(doc *goquery.Document, baseURL string, routeCaptureConf
}

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
return
}
urls[urlNoQuery] = struct{}{}
Expand Down Expand Up @@ -193,7 +193,7 @@ func ExtractLinkRoutes(doc *goquery.Document, baseURL string, routeCaptureConfig
}

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
return
}

Expand Down
10 changes: 5 additions & 5 deletions internal/discover/route/helpers/extractors/jsExtractors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func extractRoutesFromPatterns(content string, baseURL string, routeCaptureConfi
fullURL := discoverroutehelpers.ResolveURL(baseURL, urlStr)

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
continue
}

Expand Down Expand Up @@ -223,7 +223,7 @@ func extractScriptContentRoutes(ctx context.Context, scriptContent string, baseU
}

// If parsing succeeds, use AST traversal
ast.Walk(&visitor{routes: &routes, urls: urls, baseURL: baseURL, baseURLsOnly: routeCaptureConfig.IgnoreBaseUrlMatch, captureStaticAssets: routeCaptureConfig.CollectStaticAssets, errors: &errors}, program)
ast.Walk(&visitor{routes: &routes, urls: urls, baseURL: baseURL, baseURLsOnly: routeCaptureConfig.IgnoreCrossDomain, captureStaticAssets: routeCaptureConfig.CollectStaticAssets, errors: &errors}, program)
Comment thread
cursor[bot] marked this conversation as resolved.

return discoverroutehelpers.MergeWebRoutes(routes), discoverroutehelpers.SetToListString(urls), errors
}
Expand Down Expand Up @@ -277,7 +277,7 @@ func (v *visitor) processFetchCall(node *ast.CallExpression) {

// Check if the URL is allowed
// Only consider URLs that are part of the base URL if specified
if !discoverroutehelpers.IsURLAllowed(v.baseURL, urlStr, v.baseURLsOnly, v.baseURLsOnly) {
if !discoverroutehelpers.IsURLAllowed(v.baseURL, urlStr, v.baseURLsOnly, v.captureStaticAssets) {
return
}

Expand Down Expand Up @@ -348,14 +348,14 @@ func ExtractScriptRoutes(ctx context.Context, doc *goquery.Document, baseURL str
}

// If onlybaseURLs is set, only request script src that are relative
if !routeCaptureConfig.IgnoreBaseUrlMatch && discoverroutehelpers.IsAbsoluteURL(src) {
if routeCaptureConfig.IgnoreCrossDomain && discoverroutehelpers.IsAbsoluteURL(src) {
return
}

fullURL := discoverroutehelpers.ResolveURL(baseURL, src)

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, fullURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
return
}

Expand Down
5 changes: 2 additions & 3 deletions internal/discover/route/helpers/extractors/networkExtractors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (

// ExtractNetworkRoutes uses a headless browser to capture network requests and extract route details from them.
// Returns a slice of RouteDetails, a slice of URLs, and a slice of errors.
func ExtractNetworkRoutes(ctx context.Context, browser *headless.Requester, target string, baseURLsOnly bool, captureStaticAssets bool) ([]*discover.RouteDetails, []string, []string) {
func ExtractNetworkRoutes(ctx context.Context, browser *headless.Requester, target string, ignoreCrossDomain bool, captureStaticAssets bool) ([]*discover.RouteDetails, []string, []string) {
// Get the logger from the context
log := svc1log.FromContext(ctx)

Expand Down Expand Up @@ -103,8 +103,7 @@ func ExtractNetworkRoutes(ctx context.Context, browser *headless.Requester, targ
continue
}

// Skip requests that don't match the base domain when baseURLsOnly is true
if !discoverroutehelpers.IsURLAllowed(target, reqURL.String(), baseURLsOnly, captureStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(target, reqURL.String(), ignoreCrossDomain, captureStaticAssets) {
log.Debug("Skipping URL", svc1log.SafeParam("url", reqURL.String()), svc1log.SafeParam("target", target))
continue
}
Expand Down
8 changes: 3 additions & 5 deletions internal/discover/route/helpers/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,19 +269,17 @@ func URLRemoveQueryParams(rawURL string) (string, error) {
}

// IsURLAllowed checks if a target URL is allowed based on base URL, static asset rules, and domain matching.
func IsURLAllowed(baseURL string, targetURL string, ignoreBaseURLMatch bool, collectStaticAssets bool) bool {
// First check to see if the targetURL is a static asset type
func IsURLAllowed(baseURL string, targetURL string, ignoreCrossDomain bool, collectStaticAssets bool) bool {
if collectStaticAssets {
if utils.IsStaticAsset(targetURL) {
return true // Allow static assets when ignoreStaticAssets is false
return true
}
}

if ignoreBaseURLMatch {
if !ignoreCrossDomain {
return true
}

// Check if targetDomain is the same as baseDomain or a subdomain
return IsSubdomain(baseURL, targetURL)
}

Expand Down
6 changes: 3 additions & 3 deletions internal/discover/route/route.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func ExtractRedirectRoutes(redirectChain []string, baseURL string, routeCaptureC
}

// Check if the URL is allowed
if !discoverroutehelpers.IsURLAllowed(baseURL, redirectURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets) {
if !discoverroutehelpers.IsURLAllowed(baseURL, redirectURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets) {
continue
}

Expand Down Expand Up @@ -250,7 +250,7 @@ func extractRoutes(ctx context.Context, httpRequestResponse *common.HttpRequestR
}

fullRedirectedURL := redirectedURL
networkRoutes, networkUrls, networkErrors := capturerouteextractors.ExtractNetworkRoutes(networkRouteCtx, browser, fullRedirectedURL, routeCaptureConfig.IgnoreBaseUrlMatch, routeCaptureConfig.CollectStaticAssets)
networkRoutes, networkUrls, networkErrors := capturerouteextractors.ExtractNetworkRoutes(networkRouteCtx, browser, fullRedirectedURL, routeCaptureConfig.IgnoreCrossDomain, routeCaptureConfig.CollectStaticAssets)
routes = append(routes, networkRoutes...)
urls = discoverroutehelpers.AddListToSetString(urls, networkUrls)
errors = append(errors, networkErrors...)
Expand All @@ -264,7 +264,7 @@ func extractRoutes(ctx context.Context, httpRequestResponse *common.HttpRequestR
staticAssets := make(map[string]struct{})
for url := range urls {
if routeCaptureConfig.CollectStaticAssets && utils.IsStaticAsset(url) {
if discoverroutehelpers.IsSubdomain(redirectedURLBase, url) || routeCaptureConfig.IgnoreBaseUrlMatch {
if discoverroutehelpers.IsSubdomain(redirectedURLBase, url) || !routeCaptureConfig.IgnoreCrossDomain {
staticAssets[url] = struct{}{}
}
}
Expand Down
2 changes: 1 addition & 1 deletion utils/request/headless/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func handleNavigation(ctx context.Context, page *rod.Page, redirectChain *[]stri
locationURL := location.Str()
log.Debug("Captured HTTP redirect", svc1log.SafeParam("from", e.Response.URL), svc1log.SafeParam("to", locationURL), svc1log.SafeParam("status", e.Response.Status))

// Add the redirect destination to the chain if not already present
// Add the redirect destination to the chain if not already present
exists := false
for _, url := range *redirectChain {
if url == locationURL {
Expand Down
Loading