binareio · farkaskid1 · Sep 20, 2023
diff --git a/config/requirements.txt b/config/requirements.txt
@@ -0,0 +1,12 @@
+alembic==1.4.3
+SQLAlchemy==1.3.24
+psycopg2-binary==2.8.5
+requests==2.23.0
+tqdm==4.64.1
+pytz==2019.1
+pydantic==1.10.4
+fastapi==0.92.0
+uvicorn==0.11.3
+xmltodict==0.12.0
+pydantic-argparse==0.7.0
+argcomplete==2.0.0
diff --git a/config/setenv.sh b/config/setenv.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+#------------------------------------------------------------------------------
+# Script to export all needed env variables
+#------------------------------------------------------------------------------
+# Here we should add all definitions of parameters for application which are env
+# dependent. 
+# This script has to be sourced once env is started (i.e. from ${HOME}/.profile)
+# Input parameters:
+#    1) --env INP_ENV_NAME [i.e. values:dev/test/prod]
+#    2) --config INP_CFG_LIST [additional list of includes/application/module]
+#
+# Copyright (c) 2020 to date, Binare Oy ([email protected]) All rights reserved.
+#------------------------------------------------------------------------------
+getOptions()
+#------------------------------------------------------------------------------
+{
+    while [ -n "$1" ];
+    do
+        OPT="$1"
+        case ${OPT} in
+        --config)
+            shift
+            [ -n "$1" ] && INP_CFG_LIST=$1 && shift
+            ;;
+        --env)
+            shift
+            [ -n "$1" ] && export INP_ENV_NAME=$1 && shift
+            ;;
+        --home)
+            shift
+            [ -n "$1" ] && FCDB_HOME=$1 && shift
+            ;;
+        *)
+            shift
+            ;;
+        esac
+    done
+    return 0
+}
+
+#--------------------------------------------------------------------------------------------------
+#       Parse input arguments if any
+#--------------------------------------------------------------------------------------------------
+getOptions "$@"
+
+#--------------------------------------------------------------------------------------------------
+#       Main Section
+#--------------------------------------------------------------------------------------------------
+
+export FCDB_PRJ_NAME=fastcve
+
+# root path for APPIOTS project
+export FCDB_HOME=${FCDB_HOME:-${HOME}/projects/${FCDB_PRJ_NAME}}
+
+# path where configuration files are placed
+export FCDB_CFG_PATH=${FCDB_HOME}/config
+
+# main configuration file
+export FCDB_CFG_FILE=${FCDB_CFG_PATH}/setenv/config.ini
+
+# path where configuration files for additional loggers are placed
+export FCDB_CFG_LOG_PATH=${FCDB_CFG_PATH}/log
+
+# path where application log files are created/logged
+export FCDB_LOG_PATH=${FCDB_HOME}/logs
+
+# define python path
+export PYTHONPATH=${FCDB_HOME}
+
+if [ ! -d "${FCDB_LOG_PATH}" -a ! -r "${FCDB_LOG_PATH}" ]; then
+    mkdir ${FCDB_LOG_PATH}
+fi
+
+#--------------------------------------------------------------------------------------------------
+#       env type [dev/test/prod] dependent parameters
+#--------------------------------------------------------------------------------------------------
+# set default to dev is not specified explicitly
+if [ -z "$INP_ENV_NAME" ]; then
+    INP_ENV_NAME=dev
+fi
+
+export INP_ENV_NAME
+
+if [ -n "${INP_ENV_NAME}" ]; then
+
+    # any available ini file per env should be named as setenv_{INP_ENV_NAME}.ini and
+    # available under ${FCDB_CFG_PATH}
+    if [ -r "${FCDB_CFG_PATH}/setenv/setenv_${INP_ENV_NAME}.ini" ]; then
+
+        # any variable defined in these files should set variables names and values in following form:
+        # ENV_{NAME}={VALUE}
+        . ${FCDB_CFG_PATH}/setenv/setenv_${INP_ENV_NAME}.ini
+    fi
+fi
+
+#--------------------------------------------------------------------------------------------------
+export FCDB_STORAGE_INPUT="${ENV_FCDB_STORAGE_INPUT:-${FCDB_HOME}/input}"
+
+#--------------------------------------------------------------------------------------------------
+#       Log Config section
+#--------------------------------------------------------------------------------------------------
+export FCDB_LOG_CONSOLE_DEBUG_LVL=${FCDB_LOG_CONSOLE_DEBUG_LVL:-WARNING}
+export FCDB_LOG_FILE_DEBUG_LVL=${FCDB_LOG_FILE_DEBUG_LVL:-WARNING}
+
+env
diff --git a/config/setenv/config.ini b/config/setenv/config.ini
@@ -0,0 +1,73 @@
+;--------------------------------------------------------------------------------------------------
+;
+; Configuration File with all the configuration parameters needed for the application
+;
+; Copyright (c) 2020 to date, Binare Oy ([email protected]) All rights reserved.
+;--------------------------------------------------------------------------------------------------
+[db]
+
+; the DB connection string
+dsn = "postgresql://${FCDB_USER}:${FCDB_PASS}@${FCDB_HOST}:${FCDB_PORT}/smp-qa"
+; sets the pool of DB connections used by each worker, FCDB_DB_POOL_SIZE connection with extension to max FCDB_DB_POOL_OVERFLOW
+; i.e. in the case of 4 workers it would be able to simultaneosly handle 4*(FCDB_DB_POOL_OVERFLOW + FCDB_DB_POOL_OVERFLOW) requests at the same time
+; pool_pre_ping: https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic
+params = {"pool_size": ${FCDB_DB_POOL_SIZE}, "max_overflow": ${FCDB_DB_POOL_OVERFLOW}, "pool_pre_ping": "True"}
+
+;--------------------------------------------------------------------------------------------------
+[log]
+
+; namespace (or logger name) 
+namespace = root
+
+; debug level for above namespace
+level = ${FCDB_LOG_FILE_DEBUG_LVL} 
+
+; format the message : check https://docs.python.org/3/library/logging.html for possible values
+format.msg = %(levelname)-5.5s | %(asctime)s.%(msecs).3d | %(processName)s | %(threadName)-10.10s | %(name)s.%(funcName)s(%(lineno)d) : %(message)s
+
+; format acstime parameter 
+format.datefmt = %Y-%m-%d %H:%M:%S
+
+; log file name
+file.name = fastcve_console_%(process_name).log
+
+; maximum log file size before rotating to the next one
+file.max.size = 10 * 1024 * 1024
+
+; max number of rotated log files, 11th will be deleted
+file.max.count = 10
+
+;--------------------------------------------------------------------------------------------------
+[fetch]
+
+; NIST CVE API
+url.cve = https://services.nvd.nist.gov/rest/json/cves/2.0 
+
+; NIST CPE API
+url.cpe = https://services.nvd.nist.gov/rest/json/cpes/2.0
+
+; NIST Source API
+url.source = https://services.nvd.nist.gov/rest/json/source/2.0
+
+; MITRE CWE source file
+url.cwe = https://cwe.mitre.org/data/xml/views/2000.xml.zip
+
+; MITRE CAPEC source file
+url.capec = https://capec.mitre.org/data/xml/views/3000.xml.zip
+
+; API_KEY set the value of API key obtained from NVD
+api_key = ${NVD_API_KEY}
+
+; pause between requests
+request.pause.with_key = 1 #seconds to pause between requests
+request.pause.without_key = 6 #seconds to pause between requests
+
+; min time between syncs (sec)
+min.sync.time = 2 * 60 * 60 # sec
+
+; max days for period fetch (days)
+max.days.period = 120 
+
+; specify the timezone where your instance is running, this helps in accurate data updates
+; possible values UTC, Etc/GMT+/-[0-12] (i.e. Etc/GMT-5)
+timezone = Etc/GMT+2
diff --git a/config/setenv/setenv_dev.ini b/config/setenv/setenv_dev.ini
@@ -0,0 +1,14 @@
+#--------------------------------------------------------------------------------------------------
+#  Configuration values definition for DEV like env
+#--------------------------------------------------------------------------------------------------
+# Log related settings
+export FCDB_LOG_CONSOLE_DEBUG_LVL=DEBUG
+export FCDB_LOG_FILE_DEBUG_LVL=DEBUG
+
+#--------------------------------------------------------------------------------------------------
+# DB related settings
+export FCDB_DB_POOL_SIZE=20
+export FCDB_DB_POOL_OVERFLOW=40
+
+export FCDB_HOST=172.31.71.63
+export FCDB_PORT=5432
diff --git a/config/setenv/setenv_local.ini b/config/setenv/setenv_local.ini
@@ -0,0 +1,14 @@
+#--------------------------------------------------------------------------------------------------
+#  Configuration values definition for DEV like env
+#--------------------------------------------------------------------------------------------------
+# Log related settings
+FCDB_LOG_CONSOLE_DEBUG_LVL=DEBUG
+FCDB_LOG_FILE_DEBUG_LVL=DEBUG
+
+#--------------------------------------------------------------------------------------------------
+# DB related settings
+FCDB_DB_POOL_SIZE=5
+FCDB_DB_POOL_OVERFLOW=5
+
+export FCDB_HOST=localhost
+export FCDB_PORT=6630
diff --git a/config/setenv/setenv_prod.ini b/config/setenv/setenv_prod.ini
@@ -0,0 +1,14 @@
+#--------------------------------------------------------------------------------------------------
+#  Configuration values definition for DEV like env
+#--------------------------------------------------------------------------------------------------
+# Log related settings
+export FCDB_LOG_CONSOLE_DEBUG_LVL=WARNING
+export FCDB_LOG_FILE_DEBUG_LVL=INFO
+
+#--------------------------------------------------------------------------------------------------
+# DB related settings
+export FCDB_DB_POOL_SIZE=2
+export FCDB_DB_POOL_OVERFLOW=15
+
+export FCDB_HOST=localhost
+export FCDB_PORT=5432
diff --git a/config/setenv/setenv_staging.ini b/config/setenv/setenv_staging.ini
@@ -0,0 +1,14 @@
+#--------------------------------------------------------------------------------------------------
+#  Configuration values definition for DEV like env
+#--------------------------------------------------------------------------------------------------
+# Log related settings
+export FCDB_LOG_CONSOLE_DEBUG_LVL=DEBUG
+export FCDB_LOG_FILE_DEBUG_LVL=DEBUG
+
+#--------------------------------------------------------------------------------------------------
+# DB related settings
+export FCDB_DB_POOL_SIZE=20
+export FCDB_DB_POOL_OVERFLOW=40
+
+export FCDB_HOST=localhost
+export FCDB_PORT=6630
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -24,7 +24,7 @@ services:
 
         volumes:
             - vol_fastcve_db:/var/lib/postgresql/data
-              #- ./config/setenv/:/fastcve/config/setenv
+              - ./config/setenv/:/fastcve/config/setenv
             - ./:/tmp
 
         ports: