diff --git a/.github/workflows/mkdocs-develop.yml b/.github/workflows/mkdocs-develop.yml new file mode 100644 index 0000000..a121690 --- /dev/null +++ b/.github/workflows/mkdocs-develop.yml @@ -0,0 +1,41 @@ +name: 📖 Deploy Dev Docs +on: + push: + paths: + - "docs/**" + - "mkdocs.yml" + branches: + - develop +jobs: + build-documents: + name: Test Documentation Build + runs-on: ubuntu-latest + permissions: + contents: write # mkdocs gh-deploy requires write permissions to create a new branch + steps: + - name: Checkout + uses: actions/checkout@v4.1.6 + with: + fetch-depth: 0 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r docs/build/requirements.txt + - name: Test Build + run: mkdocs build --verbose --strict + - name: Deploy Docs + run: mkdocs gh-deploy --force diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml new file mode 100644 index 0000000..86b4b37 --- /dev/null +++ b/.github/workflows/test-docs.yml @@ -0,0 +1,37 @@ +name: 📙 Test docs +on: + pull_request: + paths: + - "docs/**" + - "mkdocs.yml" +jobs: + build-documents: + name: Test Documentation Build + runs-on: ubuntu-latest + permissions: + contents: read # Not required for public repositories, but for clarity + steps: + - name: Checkout + uses: actions/checkout@v4.1.6 + with: + fetch-depth: 0 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install -r docs/build/requirements.txt + - name: Test Build + run: mkdocs build --verbose --strict diff --git a/.gitignore b/.gitignore index 479a156..a84a139 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,8 @@ go.work.sum .env/ !.env/example.env +# mkdocs +/site + # Don't ignore vendors !vendor/** \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 71ba40f..6981356 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,3 +1,6 @@ +// Package cmd implements the CobraCLI commands for the methodaws CLI. Subcommands for the CLI should all live within +// this package. Logic should be delegated to internal packages and functions to keep the CLI commands clean and +// focused on CLI I/O. package cmd import ( @@ -13,6 +16,9 @@ import ( "github.com/spf13/cobra" ) +// CodeAnalyze is the main struct for the codeanalyze CLI. It contains all of the necessary fields to run the CLI and +// all the subcommands. It is also responsible for holding the Output configuration and the Output signal. This output signal +// is used to write the output of the command to the desired output format after the command has completed. type CodeAnalyze struct { Version string RootFlags config.RootFlags @@ -23,6 +29,9 @@ type CodeAnalyze struct { SemgrepCmd *cobra.Command } +// NewCodeAnalyze creates a new CodeAnalyze struct with the given version. This struct is used throughout the execution +// of the codeanalyze CLI to hold the necessary fields and subcommands. +// We pass the version in here from the main.go file, wehre we set the version string during the build process. func NewCodeAnalyze(version string) *CodeAnalyze { codeAnalyze := CodeAnalyze{ Version: version, @@ -36,6 +45,12 @@ func NewCodeAnalyze(version string) *CodeAnalyze { return &codeAnalyze } +// InitRootCommand initializes the root command for the codeanalyze CLI. This command is the main entry point for the CLI +// the CLI and is responsible for setting up the necessary flags and subcommands. It also initializes the output +// configuration and signal for the CLI. +// Importantly, it sets up the PersistentPreRunE and PersistentPostRunE functions for the root command. These functions +// are critical in setting up the output configuration for the CLI and then writing that output to the desired output +// format. func (a *CodeAnalyze) InitRootCommand() { var outputFormat string var outputFile string diff --git a/cmd/semgrep.go b/cmd/semgrep.go index 68b6319..fdc9bf8 100644 --- a/cmd/semgrep.go +++ b/cmd/semgrep.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" ) +// InitSastCommand initializes the static analysis family of commands func (a *CodeAnalyze) InitSastCommand() { a.SemgrepCmd = &cobra.Command{ Use: "semgrep", diff --git a/docs/build/Dockerfile b/docs/build/Dockerfile new file mode 100644 index 0000000..73ea9ab --- /dev/null +++ b/docs/build/Dockerfile @@ -0,0 +1,3 @@ +FROM squidfunk/mkdocs-material:9.4.6 +COPY requirements.txt . +RUN pip install -r requirements.txt diff --git a/docs/build/requirements.txt b/docs/build/requirements.txt new file mode 100644 index 0000000..de5076c --- /dev/null +++ b/docs/build/requirements.txt @@ -0,0 +1,3 @@ +mkdocs==1.6.0 +mkdocs-material==9.5.26 +Jinja2==3.1.4 \ No newline at end of file diff --git a/docs/community/community.md b/docs/community/community.md new file mode 100644 index 0000000..3a98b9a --- /dev/null +++ b/docs/community/community.md @@ -0,0 +1,7 @@ +# Contributing + +For more information on how to get involved in the Method community, please see our organization wide documentation: + +- [Discussions](https://method-security.github.io/community/contribute/discussions.html) +- [Issues](https://method-security.github.io/community/contribute/issues.html) +- [Pull Requests](https://method-security.github.io/community/contribute/pr.html) diff --git a/docs/development/adding.md b/docs/development/adding.md new file mode 100644 index 0000000..410e4ca --- /dev/null +++ b/docs/development/adding.md @@ -0,0 +1,10 @@ +# Adding a new capability + +By design, networkscan breaks every unique network scan into its own top level command. If you are looking to add a brand new capability to the tool, you can take the following steps. + +1. Add a file to `cmd/` that corresponds to the sub-command name you'd like to add to the `networkscan` CLI +2. You can use `cmd/portscan.go` as a template +3. Your file needs to be a member function of the `networkscan` struct and should be of the form `InitCommand` +4. Add a new member to the `networkscan` struct in `cmd/root.go` that corresponsds to your command name. Remember, the first letter must be capitalized. +5. Call your `Init` function from `main.go` +6. Add logic to your commands runtime and put it in its own package within `internal` (e.g., `internal/portscan`) diff --git a/docs/development/principles.md b/docs/development/principles.md new file mode 100644 index 0000000..18458fc --- /dev/null +++ b/docs/development/principles.md @@ -0,0 +1,11 @@ +# Project Principles + +## Pre-run -> Run -> Post-run + +In the root command, we set a `PersistentPreRunE` and `PersistentPostRunE` function that is responsible for initializing the output format and Signal data (in the pre-run) and then write that data in the proper format (in the post-run). + +Within the Run command that every command must implement, the output of the collected data needs to be written back to the struct's `OutputSignal.Content` value in order to be properly written out to the caller. + +## Cmd vs Internal + +By design, the functionality within each command should focus around parsing the variety of flags and options that the command may need to control capability, passing off all real logic into internal modules. diff --git a/docs/development/setup.md b/docs/development/setup.md new file mode 100644 index 0000000..5b5510b --- /dev/null +++ b/docs/development/setup.md @@ -0,0 +1,43 @@ +# Development Setup + +## Adding a new capability + +To add a new scan to networkscan, providing new enumeration capabilities to security operators everywhere, please see the [adding a new capability](./adding.md) page. + +## Setting up your development environment + +If you've just cloned networkscan for the first time, welcome to the community! We use Palantir's [godel](https://github.com/palantir/godel) to streamline local development and [goreleaser](https://goreleaser.com/) to handle the heavy lifting on the release process. + +To get started with godel, you can run + +```bash +./godelw verify +``` + +This will run a number of checks for us, including linters, tests, and license checks. We run this command as part of our CI pipeline to ensure the codebase is consistently passing tests. + +## Building the CLI + +We can use godel to build our CLI locally by running + +```bash +./godelw build +``` + +You should see output in `out/build/networkscan//-/networkscan`. + +If you'd like to clean this output up, you can run + +```bash +./godelw clean +``` + +## Testing releases locally + +We can use goreleaser locally as well to test our builds. As networkscan uses [cosign](https://github.com/sigstore/cosign) to sign our artifacts and Docker containers during our CI pipeline, we'll want to skip this step when running locally. + +```bash +goreleaser release --snapshot --clean --skip sign +``` + +This should output binaries, distributable tarballs/zips, as well as docker images to your local machine's Docker registry. diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..ed27012 --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,26 @@ +# Capabilities + +codeanalyze offers a variety of static code analysis tools and techniques that allow security teams to bring static code analysis into their automated workflows. Each of the below pages offers you an in depth look at a codeanalyze capability related to a static code analysis technique. + +- [Semgrep](./semgrep.md) + +## Top Level Flags + +networkscan has several top level flags that can be used on any subcommand. These include: + +```bash +Flags: + -h, --help help for codeanalyze + -o, --output string Output format (signal, json, yaml). Default value is signal (default "signal") + -f, --output-file string Path to output file. If blank, will output to STDOUT + -q, --quiet Suppress output + -v, --verbose Verbose output +``` + +## Version Command + +Run `networkscan version` to get the exact version information for your binary + +## Output Formats + +For more information on the various output formats that are supported by networkscan, see the [Output Formats](https://method-security.github.io/docs/output.html) page in our organization wide documentation. diff --git a/docs/docs/semgrep.md b/docs/docs/semgrep.md new file mode 100644 index 0000000..0c5a008 --- /dev/null +++ b/docs/docs/semgrep.md @@ -0,0 +1,32 @@ +# Semgrep + +The `codeanalyze semgrep` command provides an opinionated wrapping of [Semgrep](https://semgrep.dev) to facilitate tying static code analysis capabilities into your security automation workflows. + +## Usage + +```bash +codeanalyze semgrep --config-type template --config-value --target +``` + +## Help Test + +```bash +codeanalyze semgrep -h +Run semgrep against code directory + +Usage: + codeanalyze semgrep [flags] + +Flags: + --config-type string SAST config type (direct|template), direct to write custom config string (e.g. --config p/secrets), template to use a pre defined built-in and custom rule set combo + --config-value string SAST config value, either a string to be passed directly to semgrep CLI or a template value (e.g. secrets) + -h, --help help for semgrep + --local-rules-dir string Absolute path to local semgrep rules directory (default "/opt/method/codeanalyze/var/conf/resources/semgrep/") + --target string Local folder or file code target to scan + +Global Flags: + -o, --output string Output format (signal, json, yaml). Default value is signal (default "signal") + -f, --output-file string Path to output file. If blank, will output to STDOUT + -q, --quiet Suppress output + -v, --verbose Verbose output +``` diff --git a/docs/getting-started/basic-usage.md b/docs/getting-started/basic-usage.md new file mode 100644 index 0000000..82d0510 --- /dev/null +++ b/docs/getting-started/basic-usage.md @@ -0,0 +1,24 @@ +# Basic Usage + +## Binaries + +Running as a binary allows you to skip dealing with any container related networking issues and leverage the same network interface that the host machine is using. + +You can validate that the binary is working by scanning the publicly available `scanme.sh`. + +```bash +networkscan portscan --topports 100 --target scanme.sh +``` + +## Docker + +Running networkscan within a Docker container should typically work similarly to running directly on a host, however, occasionally there are a few things to keep in mind. + +If you're running on a Docker container on a MacOS machine and you are trying to scan a locally running service, you can leverage the `host.docker.internal` address as mentioned in the Docker documentation [here](https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host). + +```bash +docker run ghcr.io/method-security/networkscan \ + portscan \ + --topports 100 \ + --target scanme.sh +``` diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..84cf4e9 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,34 @@ +# Getting Started + +If you are just getting started with networkscan, welcome! This guide will walk you through the process of going zero to one with the tool. + +## Installation + +networkscan is provided in several convenient form factors, including statically compiled binary images on a variety of architectures as well as a Docker image for both x86 and ARM machines. + +If you do not see an architecture that you require, please open a [Discussion](https://method-security.github.io/community/contribute/discussions.html) to propose adding it. + +### Binaries + +networkscan currently supports statically compiled binaries across the following operating systems and architectures: + +| OS | Architecture | +| ------- | ------------ | +| Linux | amd64 | +| Linux | arm64 | +| MacOS | arm64 | +| Windows | amd64 | + +The latest binaries can be downloaded directly from [Github](https://github.com/Method-Security/networkscan/releases/latest). + +### Docker + +Docker images for networkscan are hosted in both Github Container Registry as well as on Docker Hub and can be pulled via: + +```bash +docker pull ghcr.io/method-security/networkscan +``` + +```bash +docker pull methodsecurity/networkscan +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..5964e5c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,56 @@ +# codeanalyze Documentation + +Hello and welcome to the codeanalyze documentation. While we always want to provide the most comprehensive documentation possible, we thought you may find the below sections a helpful place to get started. + +- The [Getting Started](./getting-started/basic-usage.md) section provides onboarding material +- The [Development](./development/setup.md) header is the best place to get started on developing on top of and with codeanalyze +- See the [Docs](./docs/index.md) section for a comprehensive rundown of codeanalyze capabilities + +# About codeanalyze + +codeanalyze provides an opinionated perspective on top of popular static analysis capabilities such as [Semgrep](https://semgrep.dev/) to provide visibility into vulnerabilities and misconfigurations that may exist in a team's code base. Designed with data-modeling and data-integration needs in mind, codeanalyze can be used on its own as an interactive CLI, orchestrated as part of a broader data pipeline, or leveraged from within the Method Platform. + +The types of scans that codeanalyze can conduct are constantly growing. For the most up to date listing, please see the documentation [here](./docs/index.md) + +To learn more about codeanalyze, please see the [Documentation site](https://method-security.github.io/codeanalyze/) for the most detailed information. + +## Quick Start + +### Get codeanalyze + +For the full list of available installation options, please see the [Installation](./getting-started/installation.md) page. For convenience, here are some of the most commonly used options: + +- `docker run methodsecurity/codeanalyze` +- `docker run ghcr.io/method-security/codeanalyze` +- Download the latest binary from the [Github Releases](https://github.com/Method-Security/codeanalyze/releases/latest) page +- [Installation documentation](./getting-started/installation.md) + +### General Usage + +```bash +codeanalyze semgrep +``` + +#### Examples + +```bash +codeanalyze portscan --topports 100 scanme.sh +``` + +## Contributing + +Interested in contributing to codeanalyze? Please see our organization wide [Contribution](https://method-security.github.io/community/contribute/discussions.html) page. + +## Want More? + +If you're looking for an easy way to tie codeanalyze into your broader cybersecurity workflows, or want to leverage some autonomy to improve your overall security posture, you'll love the broader Method Platform. + +For more information, visit us [here](https://method.security) + +## Community + +codeanalyze is a Method Security open source project. + +Learn more about Method's open source source work by checking out our other projects [here](https://github.com/Method-Security) or our organization wide documentation [here](https://method-security.github.io). + +Have an idea for a Tool to contribute? Open a Discussion [here](https://github.com/Method-Security/Method-Security.github.io/discussions). diff --git a/internal/config/config.go b/internal/config/config.go index e3f8730..abf0b89 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,3 +1,4 @@ +// Package config contains common configuration values that are used by the various commands and subcommands in the CLI. package config type RootFlags struct { diff --git a/internal/config/initialization.go b/internal/config/initialization.go index a5e8e96..8996307 100644 --- a/internal/config/initialization.go +++ b/internal/config/initialization.go @@ -9,6 +9,11 @@ import ( "github.com/spf13/cobra" ) +// InitializeLogging initializes the logging configuration for the CLI. It sets the log level based on the verbose and +// quiet flags provided by the user. If the quiet flag is set, the logger is set to discard all logs. If the verbose flag +// is set, the log level is set to debug. Otherwise, the log level is set to info. The logger is then returned for use +// by caller. +// We are using Palantir's Witchcraft logging library to structure our logging output. func InitializeLogging(cmd *cobra.Command, rootFlags *RootFlags) svc1log.Logger { logLevel := wlog.InfoLevel if rootFlags.Verbose { diff --git a/internal/semgrep/semgrep.go b/internal/semgrep/semgrep.go index 8dc6525..43b5991 100644 --- a/internal/semgrep/semgrep.go +++ b/internal/semgrep/semgrep.go @@ -1,3 +1,4 @@ +// Package semgrep holds all of the data structures and logic related to running semgrep commands on a codebase. package semgrep import ( @@ -8,6 +9,7 @@ import ( "strings" ) +// Metadata holds the metadata for a semgrep finding. type Metadata struct { Category string `json:"category" yaml:"category"` Confidence string `json:"confidence" yaml:"confidence"` @@ -24,6 +26,7 @@ type Metadata struct { VulnerabilityClass []string `json:"vulnerability_class" yaml:"vulnerability_class"` } +// Dev holds all of the development information for a semgrep finding. type Dev struct { Origin string `json:"origin" yaml:"origin"` RID string `json:"r_id" yaml:"r_id"` @@ -33,6 +36,7 @@ type Dev struct { VersionID string `json:"version_id" yaml:"version_id"` } +// Extra holds additional information for a semgrep finding. type Extra struct { EngineKind string `json:"engine_kind" yaml:"engine_kind"` Fingerprint string `json:"fingerprint" yaml:"fingerprint"` @@ -49,6 +53,7 @@ type Extra struct { SourceRuleURL string `json:"source-rule-url" yaml:"source-rule-url"` } +// Metavar holds the meta variable information for a semgrep finding. type Metavar struct { AbstractContent string `json:"abstract_content" yaml:"abstract_content"` End struct { @@ -63,6 +68,7 @@ type Metavar struct { } `json:"start" yaml:"start"` } +// Result holds the output result information for a given semgrep finding. type Result struct { CheckID string `json:"check_id" yaml:"check_id"` Path string `json:"path" yaml:"path"` @@ -79,6 +85,7 @@ type Result struct { Extra Extra `json:"extra" yaml:"extra"` } +// Error holds the error information for a semgrep finding. type Error struct { Code int `json:"code" yaml:"code"` Level string `json:"level" yaml:"level"` @@ -86,11 +93,14 @@ type Error struct { Type string `json:"type" yaml:"type"` } +// Report holds all of the information for a semgrep run, including all of the non-fatal errors and results. type Report struct { Errors []Error `json:"errors" yaml:"errors"` Results []Result `json:"results" yaml:"results"` } +// ExecuteSemgrep runs the semgrep command with the provided target and configValue. It returns the report of the semgrep +// including all of the results and the non-fatal errors. func ExecuteSemgrep(ctx context.Context, target string, configValue string) (Report, error) { // Prepare the semgrep arguments args := strings.Fields(configValue) diff --git a/internal/semgrep/templates.go b/internal/semgrep/templates.go index 7c12223..e7a9127 100644 --- a/internal/semgrep/templates.go +++ b/internal/semgrep/templates.go @@ -8,6 +8,8 @@ var semgrepTemplatesMap = map[string]string{ "secrets": "--config p/secrets --config %s/generic/secrets/", } +// CreateSemgrepConfigValue creates a semgrep config value based on the provided template and rules directory. If the +// template is not valid, an error is returned. func CreateSemgrepConfigValue(template string, rulesDir string) (string, error) { if templateValue, exists := semgrepTemplatesMap[template]; exists { return fmt.Sprintf(templateValue, rulesDir), nil diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..0708047 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,97 @@ +site_name: codeanalyze +site_url: https://method-security.github.io/codeanalyze/ +site_description: Static code analysis for security teams of all sizes +docs_dir: docs/ +repo_name: GitHub +repo_url: https://github.com/Method-Security/codeanalyze +edit_uri: "blob/main/docs/" + +nav: + - Getting Started: + - Overview: index.md + - Installation: getting-started/installation.md + - Basic Usage: getting-started/basic-usage.md + - Docs: + - Overview: docs/index.md + - Capabilities: + - Semgrep: docs/semgrep.md + - Contributing: + - How to contribute: community/community.md + - Development: + - Setup: development/setup.md + - Project Principles: development/principles.md + - Adding New Capabilities: development/adding.md +theme: + name: material + language: "en" + features: + - header.autohide + - navigation.tabs + - navigation.tabs.sticky + - navigation.sections + - navigation.footer + - content.action.edit + - content.tabs.link + - content.code.annotate + - content.code.copy + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + primary: blue grey + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: blue grey + toggle: + icon: material/brightness-4 + name: Switch to light mode + +markdown_extensions: + - toc: + permalink: "#" + - pymdownx.highlight + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - admonition + - footnotes + - attr_list + - pymdownx.tabbed: + alternate_style: true + - def_list + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + +extra: + generator: false + version: + method: mike + provider: mike + default: latest + social: + - icon: fontawesome/brands/twitter + link: https://twitter.com/method_security + - icon: fontawesome/brands/linkedin + link: https://www.linkedin.com/company/method-security + - icon: fontawesome/brands/github + link: https://github.com/Method-Security + + +plugins: + - search + - offline \ No newline at end of file