Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
cd56769
Add EPSS score to the CVE info
narcissus1394 Feb 15, 2024
13d2448
Add EPSS score to the CVE info_ import gzip was missing
narcissus1394 Feb 15, 2024
266b64f
search epss as an attribute of cve and fix some errors
narcissus1394 Feb 16, 2024
bee5769
search epss as an attribute of cve (remove epss from opts.searchInfo)
narcissus1394 Feb 16, 2024
d217440
update Vuln_table instead of delete/insert, run it once after save_csv
narcissus1394 Feb 19, 2024
acab011
log files are removed!
narcissus1394 Feb 19, 2024
0f62eb0
Delete env_fastcve directory
narcissus1394 Feb 20, 2024
08d4862
update load based on comments and remove unnecessary files
narcissus1394 Feb 22, 2024
f59483c
Merge remote-tracking branch 'origin/epss-score' into epss-score
narcissus1394 Feb 22, 2024
7e839d9
convert cve.metrics.epss to data['mertics']['epss'] in load and search
narcissus1394 Feb 23, 2024
482ac04
changing back to method whitelist
narcissus1394 Feb 26, 2024
132eb39
update config.ini
narcissus1394 Feb 28, 2024
4ecb3a6
remove epss from cve load, increase batch_size for epss, let epss loa…
narcissus1394 Mar 1, 2024
642f394
Addressing the comments
narcissus1394 Mar 2, 2024
e3d53ab
remove cve_called
narcissus1394 Mar 2, 2024
ca3c52a
New approach (savinf epss data to Epss table) considering new require…
narcissus1394 Mar 11, 2024
6637db2
adding the alembic script
narcissus1394 Mar 12, 2024
6bb6104
improve save_epss_data and fecth_epss_data
narcissus1394 Mar 13, 2024
d34b5e3
fixing an error in load
narcissus1394 Mar 21, 2024
c71e3d0
updating fetch_epss_data
narcissus1394 Mar 21, 2024
93ba4b6
removing alembic_init_done, adding setenv_dev.ini
narcissus1394 Mar 25, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 0 additions & 117 deletions .gitignore

This file was deleted.

4 changes: 4 additions & 0 deletions src/common/models/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,10 @@ class SearchOptions(BaseModel):
days: Optional[int] = Field(default=None, description="Number of days back when the CVEs were last modified", alias="days-back", ge=0)
deprecated: Optional[bool] = Field(default=False, description="If set to true, will fetch only the deprecated CPE names", alias="deprecated")
profile: Optional[bool] = Field(default=None, description="Would also run the profile execution of the search and save the results in a file")
epssScoreGt: Optional[float] = Field(default=None, description="Filter by EPSS score greater than", alias="epss-score-gt", gt=0) # New field for EPSS score greater than
epssScoreLt: Optional[float] = Field(default=None, description="Filter by EPSS score less than", alias="epss-score-lt", gt=0) # New field for EPSS score less than
epssPercGt: Optional[float] = Field(default=None, description="Filter by EPSS percentile greater than", alias="epss-perc-gt", ge=0, le=1) # New field for EPSS percentile greater than
epssPercLt: Optional[float] = Field(default=None, description="Filter by EPSS percentile less than", alias="epss-perc-lt", ge=0, le=1) # New field for EPSS percentile less than
output: OutputType = Field(default=OutputType.json, description="Define the output format")

class Config:
Expand Down
13 changes: 12 additions & 1 deletion src/common/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import re
import json
from typing import List, Iterator
from sqlalchemy import Boolean
from sqlalchemy import Boolean, cast, Numeric
from sqlalchemy.sql import text, expression
from sqlalchemy.orm import aliased
from generic import ApplicationContext
Expand Down Expand Up @@ -40,6 +40,17 @@ def search_cves(appctx: ApplicationContext, opts: SearchOptions):

# prepare the search query
query = session.query(cve_table)
# Filter by EPSS score
if opts.epssScoreGt is not None:
query = query.filter(cast(cve_table.data['metrics']['epss']['score'].astext, Numeric) > opts.epssScoreGt)
if opts.epssScoreLt is not None:
query = query.filter(cast(cve_table.data['metrics']['epss']['score'].astext, Numeric) < opts.epssScoreLt)

# Filter by EPSS percentile
if opts.epssPercGt is not None:
query = query.filter(cast(cve_table.data['metrics']['epss']['percentile'].astext, Numeric) > opts.epssPercGt)
if opts.epssPercLt is not None:
query = query.filter(cast(cve_table.data['metrics']['epss']['percentile'].astext, Numeric) < opts.epssPercLt)

# filter by the cve IDS, either directly specified in the search options
if opts.cveId:
Expand Down
5 changes: 4 additions & 1 deletion src/config/setenv/config.ini
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ file.max.count = 10
[fetch]

; NIST CVE API
url.cve = https://services.nvd.nist.gov/rest/json/cves/2.0
url.cve = https://services.nvd.nist.gov/rest/json/cves/2.0

; NIST CPE API
url.cpe = https://services.nvd.nist.gov/rest/json/cpes/2.0
Expand All @@ -55,6 +55,9 @@ url.cwe = https://cwe.mitre.org/data/xml/views/2000.xml.zip
; MITRE CAPEC source file
url.capec = https://capec.mitre.org/data/xml/views/3000.xml.zip

; Cynetia EPSS source file
url.epss = https://epss.cyentia.com

; API_KEY set the value of API key obtained from NVD
api_key = ${NVD_API_KEY}

Expand Down
1 change: 1 addition & 0 deletions src/config/setenv/setenv_dev.ini
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ export FCDB_DB_POOL_OVERFLOW=5

export FCDB_HOST=localhost
export FCDB_PORT=5432

42 changes: 42 additions & 0 deletions src/db/scripts/versions/9a14a98e9e6d_add_epss_table.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""Add EPSS table

Revision ID: 9a14a98e9e6d
Revises: 2f14a6a5afe8
Create Date: 2024-03-12 13:30:54.706771

"""
from alembic import op
import sqlalchemy as sa


# revision identifiers, used by Alembic.
revision = '9a14a98e9e6d'
down_revision = '2f14a6a5afe8'
branch_labels = None
depends_on = None


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('epss',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('cve_id', sa.String(length=20), nullable=True, comment='The ID of the CVE'),
sa.Column('epss_score', sa.Float(), nullable=False, comment='the score of the epss'),
sa.Column('percentile', sa.Float(), nullable=False, comment='the percentile of the epss'),
sa.Column('date', sa.DateTime(), nullable=False, comment='Date when the EPSS record has been downloaed'),
sa.Column('changed', sa.Boolean(), nullable=True, comment='indicate if epss_score has been changed'),
sa.PrimaryKeyConstraint('id'),
comment='Table that contains the list of EPSS'
)
op.create_index('epss_idx1', 'epss', ['cve_id'], unique=False)
op.create_index(op.f('ix_epss_cve_id'), 'epss', ['cve_id'], unique=False)
# ### end Alembic commands ###


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index(op.f('ix_epss_cve_id'), table_name='epss')
op.drop_index('epss_idx1', table_name='epss')
op.drop_table('epss')
# ### end Alembic commands ###

16 changes: 15 additions & 1 deletion src/db/tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"""

# coding: utf-8
from sqlalchemy import (Column, DateTime, Integer, String, Boolean, text,
from sqlalchemy import (Column, DateTime, Integer, String, Boolean, Float, text,
UniqueConstraint, Index, Text)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.dialects.postgresql.json import JSONB
Expand Down Expand Up @@ -145,3 +145,17 @@ class Capec(Base):
description = Column(Text, comment=u"the description of the CAPEC")
data = Column(JSONB, comment=u'CAPEC JSON representation')

# ------------------------------------------------------------------------------
class Epss(Base):
__tablename__ = 'epss'
__table_args__ = (
Index('epss_idx1', 'cve_id'),
{u'comment': u'Table that contains the list of EPSS'}
)

id = Column(Integer, primary_key=True)
cve_id = Column(String(20), index=True, comment=u'The ID of the CVE')
epss_score = Column(Float, nullable=False, comment=u"the score of the epss")
percentile = Column(Float, nullable=False, comment=u"the percentile of the epss")
date = Column(DateTime, nullable=False, comment=u"Date when the EPSS record has been downloaed")
changed = Column(Boolean, comment=u'indicate if epss_score has been changed')
Loading