Description

This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the package versions
of distributed files as well as the total disk size.
In addition, the conda environment used for assembling GMSPython
is checked for known vulnerabilities using pip-audit.

Contributor: Clemens Westphal, April 2020

Category : GAMS Data Utilities library

Main file : GMSPythonCheck.gms   includes :  GMSPythonCheck.gms

$title 'Consistency and vulnerability check for GMSPython' (GMSPYTHONCHECK,SEQ=140)

$onText
This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the package versions
of distributed files as well as the total disk size.
In addition, the conda environment used for assembling GMSPython
is checked for known vulnerabilities using pip-audit.

Contributor: Clemens Westphal, April 2020
$offText

$dropEnv PYTHONUSERBASE

* check disk size
$onEmbeddedCode Python:
import sys
import platform
import os
def calcSize(path):
    total = 0
    for r, d, files in os.walk(path):
        if '__pycache__' in d:
            d.remove('__pycache__')
        for f in files:
            f = os.path.join(r, f)
            if not os.path.islink(f): # skip symlinks since os.path.getsize() returns the size of the file behind the link
                total += os.path.getsize(f)
    return total
$offEmbeddedCode

$onEmbeddedCode Python:
gmsPyDir = r'%gams.sysdir%GMSPython'
errors = []
if os.path.isdir(gmsPyDir):
    if platform.system() == 'Windows':
        expectedSize = 249000000
    elif platform.system() == 'Linux':
        expectedSize = 322000000
    elif platform.system() == 'Darwin':
        if platform.machine() == 'x86_64':
            expectedSize = 331000000
        else:
            expectedSize = 220000000
    
    SizeLB = expectedSize*0.8
    SizeUB = expectedSize*1.2
    
    files = []
    for r, d, f in os.walk(gmsPyDir):
        if '__pycache__' in d:
            d.remove('__pycache__')
        files.append(f)
    files = [f for l in files for f in l]
    
    size = calcSize(gmsPyDir)
    
    if size < SizeLB or size > SizeUB:
        errors.append("Expected size of GMSPython to be between " + str(SizeLB) + " and " + str(SizeUB) + " but got " + str(size))
$offEmbeddedCode


* check redistributed packages
$onEmbeddedCode Python:
pyVersionExpected = '3.12.5'
pyVersion = str(sys.version_info.major) + '.' + str(sys.version_info.minor) + '.' + str(sys.version_info.micro)
if pyVersion != pyVersionExpected:
    errors.append("Expected Python version to be '{}', but found '{}'".format(pyVersionExpected, pyVersion))

with open(os.path.join(gmsPyDir, 'requirements.txt')) as f:
    for l in f.read().splitlines():
        m, v = l.split('==') # m=module, v=version
        m = m.lower()
        if m == 'et-xmlfile':
            m = 'et_xmlfile'
        elif m == 'python-dateutil':
            m = 'dateutil'
        elif m == 'pyyaml':
            m = 'yaml'
        elif m == 'sqlalchemy-access':
            m = 'sqlalchemy_access'
        elif m == 'pywin32':
            continue  # should be pywintypes, but that doesn't have __version__
        elif m == 'typing_extensions':
            continue  # doesn't have __version__
        elif m == 'tzdata':
            continue  # tzdata is a data-only package and can not be imported
        elif m == 'psycopg2-binary':
            m = 'psycopg2'

        try:
            module = __import__(m)
        except:
            errors.append("Could not import module '{}'".format(m))
        else:
            if m == 'pyodbc':
                mod_version = module.version
            elif m == 'pymysql': # __version__ differs from VERSION
                mod_version = '.'.join([str(i) for i in module.VERSION][:3])
            else:
                mod_version = module.__version__
            if m == 'psycopg2': # '2.9.3 (dt dec pq3 ext lo64)' -> '2.9.3'
                mod_version = mod_version.split(' ')[0]
            if v != mod_version:
                errors.append("Expected '{}' version to be '{}', but found '{}'".format(m, v, module.__version__))
$offEmbeddedCode


* Report errors regarding disk size or packages
$onEmbeddedCode Python:
if errors:
    gams.printLog("\nErrors:")
    for e in errors:
        gams.printLog(e)
    raise Exception("Errors have occurred. See the list above.")
$offEmbeddedCode

* Run pip-audit to check conda environment (gmspython) used for assembling GMSPython for vulnerabilities
* skip if no miniconda
$if not setenv MINICONDA $exit
* skip if not building master or distXX and FORCEPIPAUDIT not set
$if not setenv FORCEPIPAUDIT $if not %sysenv.GBRANCHNAME% == master $if not %sysenv.GBRANCHNAME% == dist%sysenv.GVERSIONMAJOR% $exit
$onEcho > run_pip_audit.sh
fail=0
unset PYTHONUSERBASE
eval "$($MINICONDA shell.bash hook)"
env=gmspython
[ -d ${GPORTBIN}/condaenvs/$env ] || exit 0
conda activate ${GPORTBIN}/condaenvs/$env
PYTHONUSERBASE=${GPORTBIN}/condaenvs/pythonuserbase/$env pip install pip-audit==2.7.3 --no-warn-script-location --user > /dev/null 2>&1 || { echo "Problems installing pip-audit in conda environment ${env}" ; exit 1; }
python -m pip_audit --cache-dir ./pip_audit_cache/$env --verbose > pip_audit.log 2>&1
let "fail = $?"
conda deactivate
conda deactivate
cp pip_audit.log "${GTESTDIR}/${MODTESTDIR}/pip_audit.log"
if grep -i 'Dependency not found' pip_audit.log | grep -v -i 'gamsapi'; then
    let "fail = 1"
fi
exit $fail
$offEcho

$call chmod +x ./run_pip_audit.sh
$call bash ./run_pip_audit.sh
$ifE errorlevel<>0 $abort "Failures running pip-audit. Inspect pip_audit.log for details."