Description

This test ensures the correctness of the Connect agent "GDXReader/Writer".

Contributor: Michael Bussieck, March 2022

Small Model of Type : GAMS

Category : GAMS Test library

Main file : cagdxrw.gms

$title 'Test Connect agent GDXReader/Writer' (CAGDXRW,SEQ=891)

$onText
This test ensures the correctness of the Connect agent "GDXReader/Writer".

Contributor: Michael Bussieck, March 2022
$offText


$log --- Using Python library %sysEnv.GMSPYTHONLIB%

$onEcho > t.gms
Set inew(*) canning plants /
'seattle',
'san-diego' /;

Set j(*) markets /
'new-york',
'chicago',
'topeka' /;

Parameter anew(inew) capacity of plant i in cases /
'seattle' 350,
'san-diego' 600 /;

Parameter b(j) demand at market j in cases /
'new-york' 325,
'chicago' 300,
'topeka' 275 /;

Parameter d(inew,j) distance in thousands of miles /
'seattle'.'new-york' 2.5,
'seattle'.'chicago' 1.7,
'seattle'.'topeka' 1.8,
'san-diego'.'new-york' 2.5,
'san-diego'.'chicago' 1.8,
'san-diego'.'topeka' 1.4 /;

Parameter e(*) empty;

Scalar f freight in dollars per case per thousand miles / 90 /;

Variable xnew(inew,j) shipment quantities in cases /
'seattle'.'new-york'.L 50,
'seattle'.'chicago'.L 300,
'seattle'.'topeka'.M 0.036,
'san-diego'.'new-york'.L 275,
'san-diego'.'chicago'.M 0.009,
'san-diego'.'topeka'.L 275 /;

Equation supplynew(inew) observe supply limit at plant i /
'seattle'.L 350,
'seattle'.M Eps,
'seattle'.LO -Inf,
'seattle'.UP 350,
'san-diego'.L 550,
'san-diego'.LO -Inf,
'san-diego'.UP 600 /;
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% a=c gdx=ctgdxref.gdx

$onEchoV > t.gms
Set
   i(*) 'canning plants'
   j(*) 'markets';
Parameter
   a(i)   'capacity of plant i in cases'
   b(j)   'demand at market j in cases'
   d(i,j) 'distance in thousands of miles'
   e(*)   'empty';
Scalar
   f      'freight in dollars per case per thousand miles';
Variable
   x(i,j) 'shipment quantities in cases';
Equation
   supply(i) 'observe supply limit at plant i';  

$ifThenI %MODE%==gdxin
$gdxIn ctgdxref
$load i=inew j a=anew b d f x=xnew supply=supplynew
$elseIf %MODE%==ecreadall
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxref.gdx
- GAMSWriter:
    symbols:
      - name: inew
        newName: i
      - name: j
      - name: anew
        newName: a
      - name: b
      - name: d
      - name: e
      - name: f
      - name: xnew
        newName: x
      - name: supplynew
        newName: supply
- GDXWriter:
    file: ctgdxout1.gdx
    symbols:
      - name: inew
        newName: i
      - name: j
      - name: anew
        newName: a
      - name: b
      - name: d
      - name: e
      - name: f
      - name: xnew
        newName: x
      - name: supplynew
        newName: supply
$offEmbeddedCode
$else
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxref.gdx
    symbols:
      - name: inew
        newName: i
      - name: j
      - name: anew
        newName: a
      - name: b
      - name: d
      - name: e
      - name: f
      - name: xnew
        newName: x
      - name: supplynew
        newName: supply
- GAMSWriter:
    symbols: all
- GDXWriter:
    file: ctgdxout2.gdx
$offEmbeddedCode    
$endIf
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MODE=gdxin     gdx=1.gdx
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MODE=ecreadall gdx=2.gdx
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MODE=ecreadsym gdx=3.gdx

$call.checkErrorLevel gdxdiff ctgdxout1.gdx ctgdxout2.gdx > %system.NullFile%
$call.checkErrorLevel gdxdiff ctgdxout1.gdx 1.gdx         > %system.NullFile%
$call.checkErrorLevel gdxdiff ctgdxout1.gdx 2.gdx         > %system.NullFile%
$call.checkErrorLevel gdxdiff ctgdxout1.gdx 3.gdx         > %system.NullFile%
$call.checkErrorLevel gdxdump ctgdxout1.gdx > ctgdxout1.txt
$call.checkErrorLevel gdxdump ctgdxout2.gdx > ctgdxout2.txt
$call.checkErrorLevel gdxdump 1.gdx         > 1.txt
$call.checkErrorLevel gdxdump 2.gdx         > 2.txt
$call.checkErrorLevel gdxdump 3.gdx         > 3.txt
$call.checkErrorLevel diff ctgdxout1.txt ctgdxout2.txt > %system.NullFile%
$call.checkErrorLevel diff ctgdxout1.txt 1.txt         > %system.NullFile%
$call.checkErrorLevel diff ctgdxout1.txt 2.txt         > %system.NullFile%
$call.checkErrorLevel diff ctgdxout1.txt 3.txt         > %system.NullFile%

$onEchoV > t.gms
Set i /i0*i2/; Parameter p(i);
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      import gams.transfer as gt
      gt.Parameter(connect.container, 'p', ['*'], records=[['i0',1],['i1',2],['i1',3],['i2',4]])
- GDXWriter:
    file: dr.gdx
    symbols:
      - name: p
        duplicateRecords: %DR%
- PythonCode:
    code: |
      data = connect.container['p'].records.values.tolist()
      expected = [['i0',1],['i1',2],['i1',3],['i2',4]]
      if data != expected:
          raise Exception("Data is not consistent after removing duplicate")
$offEmbeddedCode
$ifThenE.NotFail not sameas(%EXPECTED%,'fail')
$gdxIn dr.gdx
$load p
abort$(p('i1')<>%EXPECTED%) p, 'p(i1) not %EXPECTED%'
$endIf.NotFail
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=first --EXPECTED=2
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=last  --EXPECTED=3
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=none  --EXPECTED=0
$call                 gams t.gms lo=%gams.lo% --DR=all   --EXPECTED=fail
$ifE errorlevel=0 $abort expect GDXWriter to fail

$onEchoV > t.gms
Set i /i0*i2/; Parameter p(i), q(i);
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      import gams.transfer as gt
      gt.Parameter(connect.container, 'p', ['*'], records=[['i0',1],['i1',2],['i1',3],['i2',4]])
      gt.Parameter(connect.container, 'q', ['*'], records=[['i0',5],['i1',6],['i1',7],['i2',8]])
- GDXWriter:
    file: dr.gdx
    duplicateRecords: %DR%
- PythonCode:
    code: |
      data_1 = connect.container['p'].records.values.tolist()
      data_2 = connect.container['q'].records.values.tolist()
      expected_1 = [['i0',1],['i1',2],['i1',3],['i2',4]]
      expected_2 = [['i0',5],['i1',6],['i1',7],['i2',8]]
      if data_1 != expected_1:
          raise Exception("Data is not consistent after removing duplicate in symbol p")
      if data_2 != expected_2:
          raise Exception("Data is not consistent after removing duplicate in symbol q")
$offEmbeddedCode
$ifThenE.NotFail not sameas(%EXPECTED1%,'fail')
$gdxIn dr.gdx
$load p, q
abort$(p('i1')<>%EXPECTED1%) p, 'p(i1) not %EXPECTED1%'
abort$(q('i1')<>%EXPECTED2%) q, 'q(i1) not %EXPECTED2%'
$endIf.NotFail
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=first --EXPECTED1=2 --EXPECTED2=6
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=last  --EXPECTED1=3 --EXPECTED2=7
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=none  --EXPECTED1=0 --EXPECTED2=0
$call                 gams t.gms lo=%gams.lo% --DR=all   --EXPECTED1=fail --EXPECTED2=fail
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$onEchoV > t.gms
Set iii /i0*i2/;
$gdxOut xxx.gdx
$unLoad
$gdxOut
$onEmbeddedCode Connect:
- GDXReader:
    file: xxx.gdx
    symbols:
      - name: %RNAME%
- GDXWriter:
    file: yyy.gdx
    symbols:
      - name: %WNAME%
        newName: KKK
$offEmbeddedCode
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=iii --WNAME=iii
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=III --WNAME=III
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=iii --WNAME=III

set i / i1*i3 / ;
set j(*,i);
parameter d(i,*,i,*);
variable x(i,i,*);
parameter t(i,*,i,*)  / i1.tr.i3.ty 10 / ;

$gdxout cagdxr_empty_df
$unload i j d x t
$gdxout

*** test expected errors ***

$if not errorFree $abort 'Errors'
$log Test newName raising an exception for existing symbol name
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: x
    - name: i
      newName: x
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test reading an existing symbol name (specific syms)
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: i
    - name: x
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: i
    - name: x
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test reading an existing symbol name (symbols: all)
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols: all
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols: all
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test invalid option for symbols scope of GDXReader to raise an exception
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: i
      inavlidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test invalid option for symbols scope of GDXWriter to raise an exception
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: i
- GDXWriter:
    file: invalid.gdx
    symbols:
      - name: i
        invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

*********************************************************

$log test GDXReader generates empty dataFrame instead of None records
$log ### Test with Set
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: j
- PythonCode:
    code: |
      data = connect.container['j'].records
      expected_cols = ['uni', 'i', 'element_text']

      if data is None or not data.empty:
        raise Exception("Expected >j< to have an empty DataFrame.")

      if (data.columns != expected_cols).any():
        raise Exception("Unexpected columns for >j<.")
$offEmbeddedCode

$log ### Test with Parameter
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: d
    - name: t
- Filter:
    name: t
    newName: t2
    labelFilters:
      - dimension: 1
        keep: ['x']
- PythonCode:
    code: |
      data = connect.container['d'].records
      expected = connect.container['t2'].records

      if data is None or not data.empty:
        raise Exception("Expected >d< to have an empty DataFrame.")

      if (data.columns != expected.columns).any():
        raise Exception("Expected >d< to have same columns as >t2<.")
$offEmbeddedCode

$log ### Test with Variable
$onEmbeddedCode Connect:
- GDXReader:
   file: cagdxr_empty_df.gdx
   symbols:
    - name: x
- PythonCode:
    code: |
      data = connect.container['x'].records
      expected_cols = ['i_0', 'i_1', 'uni_2', 'level', 'marginal', 'lower', 'upper', 'scale']

      if data is None or not data.empty:
        raise Exception("Expected >x< to have an empty DataFrame.")

      if (data.columns != expected_cols).any():
        raise Exception("Unexpected columns for >x<.")
$offEmbeddedCode

$log ### Test symbols not invalid if duplicateRecords!=all (happens if we do a copy.deepcopy() of the Connect container)
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxref.gdx
- GDXWriter:
    file: out.gdx
    duplicateRecords: first

    symbols: all
$offEmbeddedCode