Description

This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the package versions
of distributed files as well as the total disk size.
In addition, the conda environment used for assembling GMSPython
is checked for known vulnerabilities using pip-audit.

Contributor: Clemens Westphal, April 2020

Category : GAMS Data Utilities library

Main file : GMSPythonCheck.gms   includes :  GMSPythonCheck.gms

$title 'Consistency and vulnerability check for GMSPython' (GMSPYTHONCHECK,SEQ=140)

$onText
This model performs simple checks on the GMSPython
distribution shipped with GAMS regarding the package versions
of distributed files as well as the total disk size.
In addition, the conda environment used for assembling GMSPython
is checked for known vulnerabilities using pip-audit.

Contributor: Clemens Westphal, April 2020
$offText

$dropEnv PYTHONUSERBASE

* check disk size
$onEmbeddedCode Python:
import sys
import platform
import os
from importlib import metadata
def calcSize(path):
    total = 0
    for r, d, files in os.walk(path):
        if '__pycache__' in d:
            d.remove('__pycache__')
        for f in files:
            f = os.path.join(r, f)
            if not os.path.islink(f): # skip symlinks since os.path.getsize() returns the size of the file behind the link
                total += os.path.getsize(f)
    return total
$offEmbeddedCode

$onEmbeddedCode Python:
gmsPyDir = r'%gams.sysdir%GMSPython'
errors = []
if os.path.isdir(gmsPyDir):
    if platform.system() == 'Windows':
        expectedSize = 249000000
    elif platform.system() == 'Linux':
        if platform.machine() == 'x86_64':
            expectedSize = 322000000
        else:
            expectedSize = 257000000
    elif platform.system() == 'Darwin':
        if platform.machine() == 'x86_64':
            expectedSize = 331000000
        else:
            expectedSize = 220000000
    
    SizeLB = expectedSize*0.8
    SizeUB = expectedSize*1.2
    
    files = []
    for r, d, f in os.walk(gmsPyDir):
        if '__pycache__' in d:
            d.remove('__pycache__')
        files.append(f)
    files = [f for l in files for f in l]
    
    size = calcSize(gmsPyDir)
    
    if size < SizeLB or size > SizeUB:
        errors.append("Expected size of GMSPython to be between " + str(SizeLB) + " and " + str(SizeUB) + " but got " + str(size))
$offEmbeddedCode


* check redistributed packages
$onEmbeddedCode Python:
import platform
pyVersionExpected = '3.12.10'
pyVersion = str(sys.version_info.major) + '.' + str(sys.version_info.minor) + '.' + str(sys.version_info.micro)
if pyVersion != pyVersionExpected:
    errors.append(f"Expected Python version to be '{pyVersionExpected}', but found '{pyVersion}'")

PACKAGE_TO_MODULE_MAP = {
    "python_dateutil": "dateutil",
    "pyyaml": "yaml",
    "psycopg2_binary": "psycopg2",
    "pywin32": "pywintypes",
}
with open(os.path.join(gmsPyDir, 'requirements.txt')) as f:
    for l in f.read().splitlines():
        try:
            m, v = l.split("==")  # m=module, v=version
        except ValueError:
            m, link = map(str.strip, l.split("@"))  # m=module, link=link -> e.g. numpy @ link
            v = link.split("-")[1]  # e.g.: https://......./numpy-2.0.2-cp312.......

        m = m.lower().replace("-", "_")
        m_import = PACKAGE_TO_MODULE_MAP.get(m, m)

        # Can't get the version with metadata.version
        if m in ('psycopg2', 'dateutil', 'yaml', 'pywin32'):
            continue
            
        try:
            module = __import__(m_import)
        except ModuleNotFoundError:
            errors.append(f"Could not import module '{m_import}'")
        else:
            mod_version = metadata.version(m)
            if v != mod_version:
                errors.append(
                    f"Expected '{m}' version to be '{v}', but found '{mod_version}'"
                )
$offEmbeddedCode


* Report errors regarding disk size or packages
$onEmbeddedCode Python:
if errors:
    gams.printLog("\nErrors:")
    for e in errors:
        gams.printLog(e)
    raise Exception("Errors have occurred. See the list above.")
$offEmbeddedCode

* Run pip-audit to check conda environment (gmspython) used for assembling GMSPython for vulnerabilities
* skip if not building master or distXX and FORCEPIPAUDIT not set
$if not setenv FORCEPIPAUDIT $if not %sysenv.GBRANCHNAME% == master $if not %sysenv.GBRANCHNAME% == dist%sysenv.GVERSIONMAJOR% $exit
* Default behavior for Python warnings to avoid conda warnings
$dropEnv PYTHONWARNINGS
$onEcho > run_pip_audit.sh
fail=0
env=gmspython
requirements_path="${GTESTDIR}/GMSPython/requirements.txt"
if [ ! -f "${requirements_path}" ]; then
  echo "File requirements.txt does not exist."
  exit 1
fi
if [ ! -e "${GBUILDROOT}/uvx" ]; then
  exit 0
fi

${GBUILDROOT}/uvx pip-audit@2.9.0 --disable-pip --no-deps --cache-dir ./pip_audit_cache/$env --verbose -r "$requirements_path" > pip_audit.log 2>&1
let "fail = $?"
cp pip_audit.log "${GTESTDIR}/${MODTESTDIR}/pip_audit.log"
if grep -i 'Dependency not found' pip_audit.log | grep -v -i 'gamsapi'; then
    let "fail = 1"
fi
rm -rf pip_audit_cache
exit $fail
$offEcho

$call chmod +x ./run_pip_audit.sh
$call bash ./run_pip_audit.sh
$ifE errorlevel<>0 $abort "Failures running pip-audit. Inspect pip_audit.log for details."