Description

This test ensures the correctness of the Connect agent "GAMSReader/Writer".

Contributor: Michael Bussieck, March 2022

Small Model of Type : GAMS

Category : GAMS Test library

Main file : cagamsrw.gms

$title 'Test Connect agent GAMSReader/Writer' (CAGAMSRW,SEQ=890)

$onText
This test ensures the correctness of the Connect agent "GAMSReader/Writer".

Contributor: Michael Bussieck, March 2022
$offText


$log --- Using Python library %sysEnv.GMSPYTHONLIB%

$log GAMSReader not available when no GAMS context
$onEchoV > ci.yaml
- GAMSReader:
    symbols: all
$offEcho
$call gamsconnect ci.yaml > %system.NullFile% 2>&1 
$ifE errorlevel=0 $abort expect GAMSReader to fail

$log GAMSWriter not available when no GAMS context
$onEchoV > ci.yaml
- GAMSWriter:
    symbols: all
$offEcho
$call gamsconnect ci.yaml > %system.NullFile% 2>&1
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$log GAMSWriter not available for connectOut
$echo * Empty > t.gms
$call gams t.gms lo=%gams.lo% connectOut=ci.yaml
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$log GAMSReader reads no symbols without restart for connectIn
$onEchoV > ci.yaml
- GAMSReader:
    symbols: all
- GDXWriter:
    file: ctgdxout2.gdx
    symbols: all
$offEcho
$echo * Empty > t.gms
$call rm -f ctgdxout2.gdx
$call.checkErrorLevel gams t.gms lo=%gams.lo% connectIn=ci.yaml gdx=ctgdxout3.gdx
$call.checkErrorLevel gdxdiff ctgdxout2.gdx ctgdxout3.gdx > %system.NullFile%

$log GAMSReader reads symbols from restart file for connectIn
$onEchoV > ci.yaml
- GAMSReader:
    symbols:
       - name: i
       - name: a
       - name: x
       - name: supply
- GDXWriter:
    file: ctgdxout2.gdx
    symbols: all
$offEcho
$echo execute_unload 'ctgdxout3.gdx', i, a, x, supply; > t.gms
$call.checkErrorLevel gamslib -q trnsport
$call.checkErrorLevel gams trnsport lo=%gams.lo% s=00
$call rm -f ctgdxout2.gdx
$call.checkErrorLevel gams t.gms lo=%gams.lo% connectIn=ci.yaml r=00
$call.checkErrorLevel gdxdiff ctgdxout2.gdx ctgdxout3.gdx > %system.NullFile%

$log GAMSReader reads symbols for connectOut
$onEchoV > ci.yaml
- GAMSReader:
    symbols:
       - name: i
       - name: a
       - name: x
       - name: supply
- GDXWriter:
    file: ctgdxout2.gdx
    symbols: all
$offEcho
$call rm -f ctgdxout2.gdx
$echo * Empty > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% connectOut=ci.yaml r=00
$call.checkErrorLevel gdxdiff ctgdxout2.gdx ctgdxout3.gdx > %system.NullFile%

$log GAMSWriter updates symbols for connectIn
$echo set i(*) / /; parameter a(i); variable x(i,*); equation supply(i); > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% s=00
$onEchoV > ci.yaml
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: i
       - name: a
       - name: x
       - name: supply
- GAMSWriter:
    symbols: all
    mergeType: merge
$offEcho
$echo * Empty > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml gdx=ctgdxout3.gdx
$call.checkErrorLevel gdxdiff ctgdxout2.gdx ctgdxout3.gdx > %system.NullFile%

$log GAMSWriter fails without symbols to write to
$call gams t.gms lo=%gams.lo% connectIn=ci.yaml gdx=ctgdxout3.gdx
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$echo set i / 1*5 /; parameter a(i); > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% s=00
$onEchoV > ci.yaml
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: a
- GAMSWriter:
    symbols: all
    domainCheckType: %DCTYPE%
$offEcho
$echo "abort$(card(a)>0) 'expect |a|=0';" > t.gms
$log GAMSWriter and domain violations (auto=filtered) for connectIn
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --DCTYPE=default

$log GAMSWriter and domain violations (filtered) for connectIn
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --DCTYPE=filtered

$log GAMSWriter and domain violations (checked) for connectIn
$call gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --DCTYPE=checked
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$echo set i / 1*5 /; > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% s=00
$onEchoV > ci.yaml
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: i
- GAMSWriter:
    symbols: all
    mergeType: %MTYPE%
$offEcho
$echo "abort$(card(i)<>2) 'expect |i|=2', i;" > t.gms
$log GAMSWriter and merge type (auto) for connectIn
$call gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --MTYPE=default
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$log GAMSWriter and merge type (replace) for connectIn
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --MTYPE=replace

$echo "abort$(card(i)<>7) 'expect |i|=7', i;" > t.gms
$log GAMSWriter and merge type (merge) for connectIn
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml --MTYPE=merge

$onEchoV > t.gms
set i / 1*5 /; parameter a(i);
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: a
- GAMSWriter:
    symbols: all
    domainCheckType: %DCTYPE%
$offEmbeddedCode
abort$(card(a)<>0) 'expect |a|=0';
$offEcho
$log GAMSWriter and domain violations (auto=filtered) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DCTYPE=default
$log GAMSWriter and domain violations (filtered) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DCTYPE=filtered
$log GAMSWriter and domain violations (checked) for EC
$call gams t.gms lo=%gams.lo% --DCTYPE=checked
$ifE errorlevel=0 $abort expect GAMS to fail

$onEchoV > t.gms
set i / 1*5 /; parameter a(i);
$if set FILTERED $%FILTERED%
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: a
- GAMSWriter:
    symbols: all
$offEmbeddedCode
$offEcho
$log GAMSWriter and domain violations (auto=filtered) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo%
$log GAMSWriter and domain violations (filtered) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --FILTERED=onFiltered
$log GAMSWriter and domain violations (checked) for EC
$call gams t.gms lo=%gams.lo% --FILTERED=offFiltered
$ifE errorlevel=0 $abort expect GAMS to fail

$onEchoV > t.gms
set i / 1*5 /;
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: i
- GAMSWriter:
    symbols: all
    mergeType: %MTYPE%
$offEmbeddedCode
abort$(card(i)<>%NUM%) 'expect |i|=%NUM%', i;
$offEcho
$log GAMSWriter and domain violations (auto) for EC
$call gams t.gms lo=%gams.lo% --MTYPE=default
$ifE errorlevel=0 $abort expect GAMS to fail
$log GAMSWriter and merge type (replace) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MTYPE=replace --NUM=2
$log GAMSWriter and merge type (merge) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MTYPE=merge --NUM=7

$onEchoV > t.gms
set i / 1*5 /;
$if set MERGE $%MERGE%
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: i
- GAMSWriter:
    symbols: all
$offEmbeddedCode
abort$(card(i)<>%NUM%) 'expect |i|=%NUM%', i;
$offEcho
$log GAMSWriter and merge type (auto) for EC
$call gams t.gms lo=%gams.lo%
$ifE errorlevel=0 $abort expect GAMS to fail
$log GAMSWriter and merge type (replace) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MERGE=onMultiR --NUM=2
$log GAMSWriter and  merge type (merge) for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% --MERGE=onMulti --NUM=7

$onEchoV > t.gms
set i; parameter a(i<);
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: a
- GAMSWriter:
    symbols: all
$offEmbeddedCode
abort$(card(i)<>2) 'expect |i|=2', i; 
$offEcho
$log GAMSWriter and domain defining projection for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo%

$echo set i; parameter a(i<); > t.gms
$call.checkErrorLevel gams t.gms lo=%gams.lo% s=00
$onEchoV > ci.yaml
- GDXReader:
    file: ctgdxout2.gdx 
    symbols:
       - name: a
- GAMSWriter:
    symbols: all
$offEcho
$echo "abort$(card(i)<>2) 'expect |i|=2', i;" > t.gms
$log GAMSWriter and domain defining projection for connectIn
$call.checkErrorLevel gams t.gms lo=%gams.lo% r=00 connectIn=ci.yaml

$onEcho > t.gms
Set inew(*) canning plants /
'seattle',
'san-diego' /;

Set j(*) markets /
'new-york',
'chicago',
'topeka' /;

Parameter anew(*) capacity of plant i in cases /
'seattle' 350,
'san-diego' 600 /;

Parameter b(*) demand at market j in cases /
'new-york' 325,
'chicago' 300,
'topeka' 275 /;

Parameter d(*,*) distance in thousands of miles /
'seattle'.'new-york' 2.5,
'seattle'.'chicago' 1.7,
'seattle'.'topeka' 1.8,
'san-diego'.'new-york' 2.5,
'san-diego'.'chicago' 1.8,
'san-diego'.'topeka' 1.4 /;

Scalar f freight in dollars per case per thousand miles / 90 /;

positive Variable xnew(*,*) shipment quantities in cases /
'seattle'.'new-york'.L 50,
'seattle'.'chicago'.L 300,
'seattle'.'topeka'.M 0.036,
'san-diego'.'new-york'.L 275,
'san-diego'.'chicago'.M 0.009,
'san-diego'.'topeka'.L 275 /;

Equation supplynew(*) observe supply limit at plant i /
'seattle'.L 350,
'seattle'.M Eps,
'seattle'.LO -Inf,
'seattle'.UP 350,
'san-diego'.L 550,
'san-diego'.LO -Inf,
'san-diego'.UP 600 /;
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% a=c gdx=ctgdxref.gdx
$onEchoV > t.gms
set i, j; parameter a(i), b(j), d(i,j); scalar f; variable x(i,j); equation supply(i);
$if set MERGE $%MERGE%
$onEmbeddedCode Connect:
- GDXReader:
    file: ctgdxref.gdx 
    symbols: all
- GDXWriter:
    file: ctgdx.gdx
    symbols:
      - name: inew
        newName: i
      - name: j
      - name: anew
        newName: a
      - name: b
      - name: d
      - name: f
      - name: xnew
        newName: x
      - name: supplynew
        newName: supply
- GAMSWriter:
    symbols:
      - name: inew
        newName: i
      - name: j
      - name: anew
        newName: a
      - name: b
      - name: d
      - name: f
      - name: xnew
        newName: x
      - name: supplynew
        newName: supply
$offEmbeddedCode
$offEcho
$log GAMSWriter renaming for EC
$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx=ctgdx2.gdx
$call.checkErrorLevel gdxdiff ctgdx.gdx ctgdx2.gdx > %system.NullFile%

$echo Scalar f / 7 /; Set i / i1*i3 /, j / i2 /; > t.gms
$call.checkErrorLevel gams t.gms lo=%GAMS.lo% gdx=input

$onEchoV > t.gms
Scalar f; Set dummy /i3,i2,i17/, i(*), j(*);

EmbeddedCode Connect:
- GDXReader:
    file: input.gdx
    symbols: [{name: f}, {name: i}, {name: j}]
- GAMSWriter:
    symbols: [{name: f}, {name: j}]
endEmbeddedCode f j
$offEcho
$log GAMSWriter at execution time with know uel
$call.checkErrorLevel gams t.gms lo=%gams.lo%

$onEchoV > t.gms
Scalar f; Set dummy /i3,i17/, i(*), j(*);

EmbeddedCode Connect:
- GDXReader:
    file: input.gdx
    symbols: [{name: f}, {name: i}, {name: j}]
- GAMSWriter:
    symbols: [{name: f}, {name: j}]
endEmbeddedCode f j
$offEcho
$log GAMSWriter at execution time with unknown uel
$call gams t.gms lo=%gams.lo%
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$onEchoV > t.gms
Scalar f; Set i(*), j(*);

EmbeddedCode Connect:
- GDXReader:
    file: input.gdx
    symbols: [{name: f}, {name: i}, {name: j}]
- GAMSWriter:
    symbols: [{name: f}]
endEmbeddedCode f
$offEcho
$log GAMSWriter at execution time without uel
$call.checkErrorLevel gams t.gms lo=%gams.lo%

$onEchoV > t.gms
Set i /i0*i2/; Parameter p(i);
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      import gams.transfer as gt
      gt.Parameter(connect.container, 'p', ['*'], records=[['i0',1],['i1',2],['i1',3],['i2',4]])
- GAMSWriter:
    symbols:
      - name: p
        duplicateRecords: %DR%
- PythonCode:
    code: |
      data = connect.container['p'].records.values.tolist()
      expected = [['i0',1],['i1',2],['i1',3],['i2',4]]
      if data != expected:
          raise Exception("Data is not consistent after removing duplicate")
$offEmbeddedCode p
$ifE not sameas(%EXPECTED%,'fail') abort$(p('i1')<>%EXPECTED%) p, 'p(i1) not %EXPECTED%'
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=first --EXPECTED=2
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=last  --EXPECTED=3
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=none  --EXPECTED=0
$call                 gams t.gms lo=%gams.lo% --DR=all   --EXPECTED=fail
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$onEchoV > t.gms
Set i /i0*i2/; Parameter p(i), q(i);
$onEmbeddedCode Connect:
- PythonCode:
    code: |
      import gams.transfer as gt
      gt.Parameter(connect.container, 'p', ['*'], records=[['i0',1],['i1',2],['i1',3],['i2',4]])
      gt.Parameter(connect.container, 'q', ['*'], records=[['i0',5],['i1',6],['i1',7],['i2',8]])
- GAMSWriter:
    symbols: all
    duplicateRecords: %DR%
- PythonCode:
    code: |
      data_1 = connect.container['p'].records.values.tolist()
      data_2 = connect.container['q'].records.values.tolist()
      expected_1 = [['i0',1],['i1',2],['i1',3],['i2',4]]
      expected_2 = [['i0',5],['i1',6],['i1',7],['i2',8]]
      if data_1 != expected_1:
          raise Exception("Data is not consistent after removing duplicate in symbol p")
      if data_2 != expected_2:
          raise Exception("Data is not consistent after removing duplicate in symbol q")
$offEmbeddedCode p, q
$ifE not sameas(%EXPECTED1%,fail) abort$(p('i1')<>%EXPECTED1%) p, 'p(i1) not %EXPECTED1%'
$ifE not sameas(%EXPECTED2%,fail) abort$(q('i1')<>%EXPECTED2%) q, 'q(i1) not %EXPECTED2%'
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=first --EXPECTED1=2 --EXPECTED2=6
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=last  --EXPECTED1=3 --EXPECTED2=7
$call.checkErrorLevel gams t.gms lo=%gams.lo% --DR=none  --EXPECTED1=0 --EXPECTED2=0
$call                 gams t.gms lo=%gams.lo% --DR=all   --EXPECTED1=fail --EXPECTED2=fail
$ifE errorlevel=0 $abort expect GAMSWriter to fail

$onEchoV > t.gms
Set iii /i0*i2/, kkk(*);
$onEmbeddedCode Connect:
- GAMSReader:
    symbols:
      - name: %RNAME%
- GAMSWriter:
    symbols:
      - name: %WNAME%
        newName: KKK
$offEmbeddedCode kkk
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=iii --WNAME=iii
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=III --WNAME=III
$call.checkErrorLevel gams t.gms lo=%gams.lo% --RNAME=iii --WNAME=III

set i / i1*i3 / ;
set j(*,i);
parameter d(i,*,i,*);
variable x(i,i,*);
parameter t(i,*,i,*)  / i1.tr.i3.ty 10 / ;

*** test expected errors ***

$if not errorFree $abort 'Errors'
$log Test newName raising an exception for existing symbol name
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: i
    - name: x
      newName: i
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test reading an existing symbol name (specific syms)
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: i
    - name: x
- GAMSReader:
   symbols:
    - name: i
    - name: x
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test reading an existing symbol name (symbols: all)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: all
- GAMSReader:
    symbols: all
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test invalid option for symbols scope of GAMSReader to raise an exception
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: i
      invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

$log Test invalid option for symbols scope of GAMSWriter to raise an exception
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: i
- GAMSWriter:
    symbols:
      - name: i
        invalidOption: invalid
$offEmbeddedCode
$if errorFree $abort 'Expect errors'
$clearErrors

*********************************************************

$log test GAMSReader generates empty dataFrame (With correct columns) instead of None records

$log ### Test with Set
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: j
- PythonCode:
    code: |
      data = connect.container['j'].records
      expected_cols = ['uni', 'i', 'element_text']

      if data is None or not data.empty:
        raise Exception("Expected >j< to have an empty DataFrame.")

      if (data.columns != expected_cols).any():
        raise Exception("Unexpected columns for >j<.")
$offEmbeddedCode

$log ### Test with Parameter
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: i
    - name: d
    - name: t
- Filter:
    name: t
    newName: t2
    labelFilters:
      - dimension: 1
        keep: ['x']
- PythonCode:
    code: |
      data = connect.container['d'].records
      expected = connect.container['t2'].records

      if data is None or not data.empty:
        raise Exception("Expected >d< to have an empty DataFrame.")

      if (data.columns != expected.columns).any():
        raise Exception("Expected >d< to have same columns as >t2<.")
$offEmbeddedCode

$log ### Test with Variable
$onEmbeddedCode Connect:
- GAMSReader:
   symbols:
    - name: x
- PythonCode:
    code: |
      data = connect.container['x'].records
      expected_cols = ['i_0', 'i_1', 'uni_2', 'level', 'marginal', 'lower', 'upper', 'scale']

      if data is None or not data.empty:
        raise Exception("Expected >x< to have an empty DataFrame.")

      if (data.columns != expected_cols).any():
        raise Exception("Unexpected columns for >x<.")
$offEmbeddedCode

$log ### Test symbols not invalid if duplicateRecords!=all (happens if we do a copy.deepcopy() of the Connect container)
$onEmbeddedCode Connect:
- GAMSReader:
    symbols: all
- GAMSWriter:
    duplicateRecords: first
    symbols: all
    mergeType: merge
$offEmbeddedCode