diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b8c4d7c..e93f607 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,5 +14,5 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} branch: main draft: false - version: 4.52.2 - description: 'Version 4.52.2 of sasjs/core is now on SAS PACKAGES :ok_hand:' + version: 4.52.3 + description: 'Version 4.52.3 of sasjs/core is now on SAS PACKAGES :ok_hand:' diff --git a/005_macros/mf_abort.sas b/001_macros/mf_abort.sas similarity index 100% rename from 005_macros/mf_abort.sas rename to 001_macros/mf_abort.sas diff --git a/005_macros/mf_dedup.sas b/001_macros/mf_dedup.sas similarity index 100% rename from 005_macros/mf_dedup.sas rename to 001_macros/mf_dedup.sas diff --git a/005_macros/mf_deletefile.sas b/001_macros/mf_deletefile.sas similarity index 100% rename from 005_macros/mf_deletefile.sas rename to 001_macros/mf_deletefile.sas diff --git a/005_macros/mf_existds.sas b/001_macros/mf_existds.sas similarity index 100% rename from 005_macros/mf_existds.sas rename to 001_macros/mf_existds.sas diff --git a/005_macros/mf_existfeature.sas b/001_macros/mf_existfeature.sas similarity index 100% rename from 005_macros/mf_existfeature.sas rename to 001_macros/mf_existfeature.sas diff --git a/005_macros/mf_existfileref.sas b/001_macros/mf_existfileref.sas similarity index 100% rename from 005_macros/mf_existfileref.sas rename to 001_macros/mf_existfileref.sas diff --git a/005_macros/mf_existfunction.sas b/001_macros/mf_existfunction.sas similarity index 100% rename from 005_macros/mf_existfunction.sas rename to 001_macros/mf_existfunction.sas diff --git a/005_macros/mf_existvar.sas b/001_macros/mf_existvar.sas similarity index 100% rename from 005_macros/mf_existvar.sas rename to 001_macros/mf_existvar.sas diff --git a/005_macros/mf_existvarlist.sas b/001_macros/mf_existvarlist.sas similarity index 100% rename from 005_macros/mf_existvarlist.sas rename to 001_macros/mf_existvarlist.sas diff --git a/005_macros/mf_fmtdttm.sas b/001_macros/mf_fmtdttm.sas similarity index 100% rename from 005_macros/mf_fmtdttm.sas rename to 001_macros/mf_fmtdttm.sas diff --git a/005_macros/mf_getapploc.sas b/001_macros/mf_getapploc.sas similarity index 100% rename from 005_macros/mf_getapploc.sas rename to 001_macros/mf_getapploc.sas diff --git a/005_macros/mf_getattrc.sas b/001_macros/mf_getattrc.sas similarity index 100% rename from 005_macros/mf_getattrc.sas rename to 001_macros/mf_getattrc.sas diff --git a/005_macros/mf_getattrn.sas b/001_macros/mf_getattrn.sas similarity index 100% rename from 005_macros/mf_getattrn.sas rename to 001_macros/mf_getattrn.sas diff --git a/005_macros/mf_getengine.sas b/001_macros/mf_getengine.sas similarity index 100% rename from 005_macros/mf_getengine.sas rename to 001_macros/mf_getengine.sas diff --git a/005_macros/mf_getfilesize.sas b/001_macros/mf_getfilesize.sas similarity index 100% rename from 005_macros/mf_getfilesize.sas rename to 001_macros/mf_getfilesize.sas diff --git a/005_macros/mf_getfmtlist.sas b/001_macros/mf_getfmtlist.sas similarity index 100% rename from 005_macros/mf_getfmtlist.sas rename to 001_macros/mf_getfmtlist.sas diff --git a/005_macros/mf_getfmtname.sas b/001_macros/mf_getfmtname.sas similarity index 100% rename from 005_macros/mf_getfmtname.sas rename to 001_macros/mf_getfmtname.sas diff --git a/005_macros/mf_getgitbranch.sas b/001_macros/mf_getgitbranch.sas similarity index 100% rename from 005_macros/mf_getgitbranch.sas rename to 001_macros/mf_getgitbranch.sas diff --git a/005_macros/mf_getkeyvalue.sas b/001_macros/mf_getkeyvalue.sas similarity index 100% rename from 005_macros/mf_getkeyvalue.sas rename to 001_macros/mf_getkeyvalue.sas diff --git a/005_macros/mf_getplatform.sas b/001_macros/mf_getplatform.sas similarity index 100% rename from 005_macros/mf_getplatform.sas rename to 001_macros/mf_getplatform.sas diff --git a/005_macros/mf_getquotedstr.sas b/001_macros/mf_getquotedstr.sas similarity index 100% rename from 005_macros/mf_getquotedstr.sas rename to 001_macros/mf_getquotedstr.sas diff --git a/005_macros/mf_getschema.sas b/001_macros/mf_getschema.sas similarity index 100% rename from 005_macros/mf_getschema.sas rename to 001_macros/mf_getschema.sas diff --git a/005_macros/mf_getuniquefileref.sas b/001_macros/mf_getuniquefileref.sas similarity index 100% rename from 005_macros/mf_getuniquefileref.sas rename to 001_macros/mf_getuniquefileref.sas diff --git a/005_macros/mf_getuniquelibref.sas b/001_macros/mf_getuniquelibref.sas similarity index 100% rename from 005_macros/mf_getuniquelibref.sas rename to 001_macros/mf_getuniquelibref.sas diff --git a/005_macros/mf_getuniquename.sas b/001_macros/mf_getuniquename.sas similarity index 100% rename from 005_macros/mf_getuniquename.sas rename to 001_macros/mf_getuniquename.sas diff --git a/005_macros/mf_getuser.sas b/001_macros/mf_getuser.sas similarity index 100% rename from 005_macros/mf_getuser.sas rename to 001_macros/mf_getuser.sas diff --git a/005_macros/mf_getvalue.sas b/001_macros/mf_getvalue.sas similarity index 100% rename from 005_macros/mf_getvalue.sas rename to 001_macros/mf_getvalue.sas diff --git a/005_macros/mf_getvarcount.sas b/001_macros/mf_getvarcount.sas similarity index 100% rename from 005_macros/mf_getvarcount.sas rename to 001_macros/mf_getvarcount.sas diff --git a/005_macros/mf_getvarformat.sas b/001_macros/mf_getvarformat.sas similarity index 100% rename from 005_macros/mf_getvarformat.sas rename to 001_macros/mf_getvarformat.sas diff --git a/005_macros/mf_getvarlen.sas b/001_macros/mf_getvarlen.sas similarity index 100% rename from 005_macros/mf_getvarlen.sas rename to 001_macros/mf_getvarlen.sas diff --git a/005_macros/mf_getvarlist.sas b/001_macros/mf_getvarlist.sas similarity index 100% rename from 005_macros/mf_getvarlist.sas rename to 001_macros/mf_getvarlist.sas diff --git a/005_macros/mf_getvarnum.sas b/001_macros/mf_getvarnum.sas similarity index 100% rename from 005_macros/mf_getvarnum.sas rename to 001_macros/mf_getvarnum.sas diff --git a/005_macros/mf_getvartype.sas b/001_macros/mf_getvartype.sas similarity index 100% rename from 005_macros/mf_getvartype.sas rename to 001_macros/mf_getvartype.sas diff --git a/005_macros/mf_getxengine.sas b/001_macros/mf_getxengine.sas similarity index 100% rename from 005_macros/mf_getxengine.sas rename to 001_macros/mf_getxengine.sas diff --git a/005_macros/mf_increment.sas b/001_macros/mf_increment.sas similarity index 100% rename from 005_macros/mf_increment.sas rename to 001_macros/mf_increment.sas diff --git a/009_macros/mf_isblank.sas b/001_macros/mf_isblank.sas similarity index 100% rename from 009_macros/mf_isblank.sas rename to 001_macros/mf_isblank.sas diff --git a/005_macros/mf_isdir.sas b/001_macros/mf_isdir.sas similarity index 100% rename from 005_macros/mf_isdir.sas rename to 001_macros/mf_isdir.sas diff --git a/005_macros/mf_isint.sas b/001_macros/mf_isint.sas similarity index 100% rename from 005_macros/mf_isint.sas rename to 001_macros/mf_isint.sas diff --git a/005_macros/mf_islibds.sas b/001_macros/mf_islibds.sas similarity index 100% rename from 005_macros/mf_islibds.sas rename to 001_macros/mf_islibds.sas diff --git a/005_macros/mf_loc.sas b/001_macros/mf_loc.sas similarity index 100% rename from 005_macros/mf_loc.sas rename to 001_macros/mf_loc.sas diff --git a/005_macros/mf_mkdir.sas b/001_macros/mf_mkdir.sas similarity index 100% rename from 005_macros/mf_mkdir.sas rename to 001_macros/mf_mkdir.sas diff --git a/005_macros/mf_mval.sas b/001_macros/mf_mval.sas similarity index 100% rename from 005_macros/mf_mval.sas rename to 001_macros/mf_mval.sas diff --git a/005_macros/mf_nobs.sas b/001_macros/mf_nobs.sas similarity index 100% rename from 005_macros/mf_nobs.sas rename to 001_macros/mf_nobs.sas diff --git a/005_macros/mf_readfile.sas b/001_macros/mf_readfile.sas similarity index 100% rename from 005_macros/mf_readfile.sas rename to 001_macros/mf_readfile.sas diff --git a/005_macros/mf_trimstr.sas b/001_macros/mf_trimstr.sas similarity index 100% rename from 005_macros/mf_trimstr.sas rename to 001_macros/mf_trimstr.sas diff --git a/005_macros/mf_uid.sas b/001_macros/mf_uid.sas similarity index 100% rename from 005_macros/mf_uid.sas rename to 001_macros/mf_uid.sas diff --git a/005_macros/mf_verifymacvars.sas b/001_macros/mf_verifymacvars.sas similarity index 100% rename from 005_macros/mf_verifymacvars.sas rename to 001_macros/mf_verifymacvars.sas diff --git a/005_macros/mf_wordsinstr1andstr2.sas b/001_macros/mf_wordsinstr1andstr2.sas similarity index 100% rename from 005_macros/mf_wordsinstr1andstr2.sas rename to 001_macros/mf_wordsinstr1andstr2.sas diff --git a/005_macros/mf_wordsinstr1butnotstr2.sas b/001_macros/mf_wordsinstr1butnotstr2.sas similarity index 100% rename from 005_macros/mf_wordsinstr1butnotstr2.sas rename to 001_macros/mf_wordsinstr1butnotstr2.sas diff --git a/005_macros/mf_writefile.sas b/001_macros/mf_writefile.sas similarity index 100% rename from 005_macros/mf_writefile.sas rename to 001_macros/mf_writefile.sas diff --git a/005_macros/mp_abort.sas b/001_macros/mp_abort.sas similarity index 100% rename from 005_macros/mp_abort.sas rename to 001_macros/mp_abort.sas diff --git a/005_macros/mp_aligndecimal.sas b/001_macros/mp_aligndecimal.sas similarity index 100% rename from 005_macros/mp_aligndecimal.sas rename to 001_macros/mp_aligndecimal.sas diff --git a/005_macros/mp_appendfile.sas b/001_macros/mp_appendfile.sas similarity index 100% rename from 005_macros/mp_appendfile.sas rename to 001_macros/mp_appendfile.sas diff --git a/005_macros/mp_applyformats.sas b/001_macros/mp_applyformats.sas similarity index 100% rename from 005_macros/mp_applyformats.sas rename to 001_macros/mp_applyformats.sas diff --git a/005_macros/mp_assert.sas b/001_macros/mp_assert.sas similarity index 100% rename from 005_macros/mp_assert.sas rename to 001_macros/mp_assert.sas diff --git a/005_macros/mp_assertcols.sas b/001_macros/mp_assertcols.sas similarity index 100% rename from 005_macros/mp_assertcols.sas rename to 001_macros/mp_assertcols.sas diff --git a/005_macros/mp_assertcolvals.sas b/001_macros/mp_assertcolvals.sas similarity index 100% rename from 005_macros/mp_assertcolvals.sas rename to 001_macros/mp_assertcolvals.sas diff --git a/005_macros/mp_assertdsobs.sas b/001_macros/mp_assertdsobs.sas similarity index 100% rename from 005_macros/mp_assertdsobs.sas rename to 001_macros/mp_assertdsobs.sas diff --git a/005_macros/mp_assertscope.sas b/001_macros/mp_assertscope.sas similarity index 100% rename from 005_macros/mp_assertscope.sas rename to 001_macros/mp_assertscope.sas diff --git a/005_macros/mp_base64copy.sas b/001_macros/mp_base64copy.sas similarity index 100% rename from 005_macros/mp_base64copy.sas rename to 001_macros/mp_base64copy.sas diff --git a/005_macros/mp_binarycopy.sas b/001_macros/mp_binarycopy.sas similarity index 100% rename from 005_macros/mp_binarycopy.sas rename to 001_macros/mp_binarycopy.sas diff --git a/005_macros/mp_chop.sas b/001_macros/mp_chop.sas similarity index 100% rename from 005_macros/mp_chop.sas rename to 001_macros/mp_chop.sas diff --git a/005_macros/mp_cleancsv.sas b/001_macros/mp_cleancsv.sas similarity index 100% rename from 005_macros/mp_cleancsv.sas rename to 001_macros/mp_cleancsv.sas diff --git a/009_macros/mp_cntlout.sas b/001_macros/mp_cntlout.sas similarity index 100% rename from 009_macros/mp_cntlout.sas rename to 001_macros/mp_cntlout.sas diff --git a/005_macros/mp_copyfolder.sas b/001_macros/mp_copyfolder.sas similarity index 100% rename from 005_macros/mp_copyfolder.sas rename to 001_macros/mp_copyfolder.sas diff --git a/005_macros/mp_coretable.sas b/001_macros/mp_coretable.sas similarity index 100% rename from 005_macros/mp_coretable.sas rename to 001_macros/mp_coretable.sas diff --git a/005_macros/mp_createconstraints.sas b/001_macros/mp_createconstraints.sas similarity index 100% rename from 005_macros/mp_createconstraints.sas rename to 001_macros/mp_createconstraints.sas diff --git a/005_macros/mp_createwebservice.sas b/001_macros/mp_createwebservice.sas similarity index 100% rename from 005_macros/mp_createwebservice.sas rename to 001_macros/mp_createwebservice.sas diff --git a/005_macros/mp_csv2ds.sas b/001_macros/mp_csv2ds.sas similarity index 100% rename from 005_macros/mp_csv2ds.sas rename to 001_macros/mp_csv2ds.sas diff --git a/005_macros/mp_deleteconstraints.sas b/001_macros/mp_deleteconstraints.sas similarity index 100% rename from 005_macros/mp_deleteconstraints.sas rename to 001_macros/mp_deleteconstraints.sas diff --git a/005_macros/mp_deletefolder.sas b/001_macros/mp_deletefolder.sas similarity index 100% rename from 005_macros/mp_deletefolder.sas rename to 001_macros/mp_deletefolder.sas diff --git a/005_macros/mp_dictionary.sas b/001_macros/mp_dictionary.sas similarity index 100% rename from 005_macros/mp_dictionary.sas rename to 001_macros/mp_dictionary.sas diff --git a/009_macros/mp_dirlist.sas b/001_macros/mp_dirlist.sas similarity index 100% rename from 009_macros/mp_dirlist.sas rename to 001_macros/mp_dirlist.sas diff --git a/005_macros/mp_distinctfmtvalues.sas b/001_macros/mp_distinctfmtvalues.sas similarity index 100% rename from 005_macros/mp_distinctfmtvalues.sas rename to 001_macros/mp_distinctfmtvalues.sas diff --git a/005_macros/mp_dropmembers.sas b/001_macros/mp_dropmembers.sas similarity index 100% rename from 005_macros/mp_dropmembers.sas rename to 001_macros/mp_dropmembers.sas diff --git a/005_macros/mp_ds2cards.sas b/001_macros/mp_ds2cards.sas similarity index 100% rename from 005_macros/mp_ds2cards.sas rename to 001_macros/mp_ds2cards.sas diff --git a/009_macros/mp_ds2csv.sas b/001_macros/mp_ds2csv.sas similarity index 100% rename from 009_macros/mp_ds2csv.sas rename to 001_macros/mp_ds2csv.sas diff --git a/005_macros/mp_ds2ddl.sas b/001_macros/mp_ds2ddl.sas similarity index 100% rename from 005_macros/mp_ds2ddl.sas rename to 001_macros/mp_ds2ddl.sas diff --git a/005_macros/mp_ds2fmtds.sas b/001_macros/mp_ds2fmtds.sas similarity index 100% rename from 005_macros/mp_ds2fmtds.sas rename to 001_macros/mp_ds2fmtds.sas diff --git a/005_macros/mp_ds2inserts.sas b/001_macros/mp_ds2inserts.sas similarity index 100% rename from 005_macros/mp_ds2inserts.sas rename to 001_macros/mp_ds2inserts.sas diff --git a/005_macros/mp_ds2md.sas b/001_macros/mp_ds2md.sas similarity index 100% rename from 005_macros/mp_ds2md.sas rename to 001_macros/mp_ds2md.sas diff --git a/005_macros/mp_ds2squeeze.sas b/001_macros/mp_ds2squeeze.sas similarity index 100% rename from 005_macros/mp_ds2squeeze.sas rename to 001_macros/mp_ds2squeeze.sas diff --git a/005_macros/mp_dsmeta.sas b/001_macros/mp_dsmeta.sas similarity index 100% rename from 005_macros/mp_dsmeta.sas rename to 001_macros/mp_dsmeta.sas diff --git a/009_macros/mp_filtercheck.sas b/001_macros/mp_filtercheck.sas similarity index 100% rename from 009_macros/mp_filtercheck.sas rename to 001_macros/mp_filtercheck.sas diff --git a/005_macros/mp_filtergenerate.sas b/001_macros/mp_filtergenerate.sas similarity index 100% rename from 005_macros/mp_filtergenerate.sas rename to 001_macros/mp_filtergenerate.sas diff --git a/005_macros/mp_filterstore.sas b/001_macros/mp_filterstore.sas similarity index 100% rename from 005_macros/mp_filterstore.sas rename to 001_macros/mp_filterstore.sas diff --git a/005_macros/mp_filtervalidate.sas b/001_macros/mp_filtervalidate.sas similarity index 100% rename from 005_macros/mp_filtervalidate.sas rename to 001_macros/mp_filtervalidate.sas diff --git a/005_macros/mp_getcols.sas b/001_macros/mp_getcols.sas similarity index 100% rename from 005_macros/mp_getcols.sas rename to 001_macros/mp_getcols.sas diff --git a/005_macros/mp_getconstraints.sas b/001_macros/mp_getconstraints.sas similarity index 100% rename from 005_macros/mp_getconstraints.sas rename to 001_macros/mp_getconstraints.sas diff --git a/005_macros/mp_getdbml.sas b/001_macros/mp_getdbml.sas similarity index 100% rename from 005_macros/mp_getdbml.sas rename to 001_macros/mp_getdbml.sas diff --git a/005_macros/mp_getddl.sas b/001_macros/mp_getddl.sas similarity index 100% rename from 005_macros/mp_getddl.sas rename to 001_macros/mp_getddl.sas diff --git a/005_macros/mp_getformats.sas b/001_macros/mp_getformats.sas similarity index 100% rename from 005_macros/mp_getformats.sas rename to 001_macros/mp_getformats.sas diff --git a/005_macros/mp_getmaxvarlengths.sas b/001_macros/mp_getmaxvarlengths.sas similarity index 100% rename from 005_macros/mp_getmaxvarlengths.sas rename to 001_macros/mp_getmaxvarlengths.sas diff --git a/005_macros/mp_getpk.sas b/001_macros/mp_getpk.sas similarity index 100% rename from 005_macros/mp_getpk.sas rename to 001_macros/mp_getpk.sas diff --git a/005_macros/mp_gitadd.sas b/001_macros/mp_gitadd.sas similarity index 100% rename from 005_macros/mp_gitadd.sas rename to 001_macros/mp_gitadd.sas diff --git a/005_macros/mp_gitlog.sas b/001_macros/mp_gitlog.sas similarity index 100% rename from 005_macros/mp_gitlog.sas rename to 001_macros/mp_gitlog.sas diff --git a/005_macros/mp_gitreleaseinfo.sas b/001_macros/mp_gitreleaseinfo.sas similarity index 100% rename from 005_macros/mp_gitreleaseinfo.sas rename to 001_macros/mp_gitreleaseinfo.sas diff --git a/005_macros/mp_gitstatus.sas b/001_macros/mp_gitstatus.sas similarity index 100% rename from 005_macros/mp_gitstatus.sas rename to 001_macros/mp_gitstatus.sas diff --git a/005_macros/mp_gsubfile.sas b/001_macros/mp_gsubfile.sas similarity index 100% rename from 005_macros/mp_gsubfile.sas rename to 001_macros/mp_gsubfile.sas diff --git a/005_macros/mp_guesspk.sas b/001_macros/mp_guesspk.sas similarity index 100% rename from 005_macros/mp_guesspk.sas rename to 001_macros/mp_guesspk.sas diff --git a/005_macros/mp_hashdataset.sas b/001_macros/mp_hashdataset.sas similarity index 100% rename from 005_macros/mp_hashdataset.sas rename to 001_macros/mp_hashdataset.sas diff --git a/005_macros/mp_hashdirectory.sas b/001_macros/mp_hashdirectory.sas similarity index 100% rename from 005_macros/mp_hashdirectory.sas rename to 001_macros/mp_hashdirectory.sas diff --git a/005_macros/mp_include.sas b/001_macros/mp_include.sas similarity index 100% rename from 005_macros/mp_include.sas rename to 001_macros/mp_include.sas diff --git a/005_macros/mp_init.sas b/001_macros/mp_init.sas similarity index 100% rename from 005_macros/mp_init.sas rename to 001_macros/mp_init.sas diff --git a/005_macros/mp_jsonout.sas b/001_macros/mp_jsonout.sas similarity index 100% rename from 005_macros/mp_jsonout.sas rename to 001_macros/mp_jsonout.sas diff --git a/005_macros/mp_lib2cards.sas b/001_macros/mp_lib2cards.sas similarity index 100% rename from 005_macros/mp_lib2cards.sas rename to 001_macros/mp_lib2cards.sas diff --git a/005_macros/mp_lib2inserts.sas b/001_macros/mp_lib2inserts.sas similarity index 100% rename from 005_macros/mp_lib2inserts.sas rename to 001_macros/mp_lib2inserts.sas diff --git a/009_macros/mp_loadformat.sas b/001_macros/mp_loadformat.sas similarity index 100% rename from 009_macros/mp_loadformat.sas rename to 001_macros/mp_loadformat.sas diff --git a/005_macros/mp_lockanytable.sas b/001_macros/mp_lockanytable.sas similarity index 100% rename from 005_macros/mp_lockanytable.sas rename to 001_macros/mp_lockanytable.sas diff --git a/005_macros/mp_lockfilecheck.sas b/001_macros/mp_lockfilecheck.sas similarity index 100% rename from 005_macros/mp_lockfilecheck.sas rename to 001_macros/mp_lockfilecheck.sas diff --git a/005_macros/mp_makedata.sas b/001_macros/mp_makedata.sas similarity index 100% rename from 005_macros/mp_makedata.sas rename to 001_macros/mp_makedata.sas diff --git a/005_macros/mp_md5.sas b/001_macros/mp_md5.sas similarity index 100% rename from 005_macros/mp_md5.sas rename to 001_macros/mp_md5.sas diff --git a/005_macros/mp_perflog.sas b/001_macros/mp_perflog.sas similarity index 100% rename from 005_macros/mp_perflog.sas rename to 001_macros/mp_perflog.sas diff --git a/005_macros/mp_prevobs.sas b/001_macros/mp_prevobs.sas similarity index 100% rename from 005_macros/mp_prevobs.sas rename to 001_macros/mp_prevobs.sas diff --git a/005_macros/mp_recursivejoin.sas b/001_macros/mp_recursivejoin.sas similarity index 100% rename from 005_macros/mp_recursivejoin.sas rename to 001_macros/mp_recursivejoin.sas diff --git a/005_macros/mp_replace.sas b/001_macros/mp_replace.sas similarity index 100% rename from 005_macros/mp_replace.sas rename to 001_macros/mp_replace.sas diff --git a/005_macros/mp_reseterror.sas b/001_macros/mp_reseterror.sas similarity index 100% rename from 005_macros/mp_reseterror.sas rename to 001_macros/mp_reseterror.sas diff --git a/005_macros/mp_resetoption.sas b/001_macros/mp_resetoption.sas similarity index 100% rename from 005_macros/mp_resetoption.sas rename to 001_macros/mp_resetoption.sas diff --git a/005_macros/mp_retainedkey.sas b/001_macros/mp_retainedkey.sas similarity index 100% rename from 005_macros/mp_retainedkey.sas rename to 001_macros/mp_retainedkey.sas diff --git a/005_macros/mp_runddl.sas b/001_macros/mp_runddl.sas similarity index 100% rename from 005_macros/mp_runddl.sas rename to 001_macros/mp_runddl.sas diff --git a/005_macros/mp_searchcols.sas b/001_macros/mp_searchcols.sas similarity index 100% rename from 005_macros/mp_searchcols.sas rename to 001_macros/mp_searchcols.sas diff --git a/005_macros/mp_searchdata.sas b/001_macros/mp_searchdata.sas similarity index 100% rename from 005_macros/mp_searchdata.sas rename to 001_macros/mp_searchdata.sas diff --git a/005_macros/mp_setkeyvalue.sas b/001_macros/mp_setkeyvalue.sas similarity index 100% rename from 005_macros/mp_setkeyvalue.sas rename to 001_macros/mp_setkeyvalue.sas diff --git a/005_macros/mp_sortinplace.sas b/001_macros/mp_sortinplace.sas similarity index 100% rename from 005_macros/mp_sortinplace.sas rename to 001_macros/mp_sortinplace.sas diff --git a/009_macros/mp_stackdiffs.sas b/001_macros/mp_stackdiffs.sas similarity index 100% rename from 009_macros/mp_stackdiffs.sas rename to 001_macros/mp_stackdiffs.sas diff --git a/009_macros/mp_storediffs.sas b/001_macros/mp_storediffs.sas similarity index 100% rename from 009_macros/mp_storediffs.sas rename to 001_macros/mp_storediffs.sas diff --git a/005_macros/mp_stprequests.sas b/001_macros/mp_stprequests.sas similarity index 100% rename from 005_macros/mp_stprequests.sas rename to 001_macros/mp_stprequests.sas diff --git a/005_macros/mp_streamfile.sas b/001_macros/mp_streamfile.sas similarity index 100% rename from 005_macros/mp_streamfile.sas rename to 001_macros/mp_streamfile.sas diff --git a/009_macros/mp_stripdiffs.sas b/001_macros/mp_stripdiffs.sas similarity index 100% rename from 009_macros/mp_stripdiffs.sas rename to 001_macros/mp_stripdiffs.sas diff --git a/005_macros/mp_testjob.sas b/001_macros/mp_testjob.sas similarity index 100% rename from 005_macros/mp_testjob.sas rename to 001_macros/mp_testjob.sas diff --git a/005_macros/mp_testservice.sas b/001_macros/mp_testservice.sas similarity index 100% rename from 005_macros/mp_testservice.sas rename to 001_macros/mp_testservice.sas diff --git a/005_macros/mp_testwritespeedlibrary.sas b/001_macros/mp_testwritespeedlibrary.sas similarity index 100% rename from 005_macros/mp_testwritespeedlibrary.sas rename to 001_macros/mp_testwritespeedlibrary.sas diff --git a/005_macros/mp_tree.sas b/001_macros/mp_tree.sas similarity index 100% rename from 005_macros/mp_tree.sas rename to 001_macros/mp_tree.sas diff --git a/005_macros/mp_unzip.sas b/001_macros/mp_unzip.sas similarity index 100% rename from 005_macros/mp_unzip.sas rename to 001_macros/mp_unzip.sas diff --git a/005_macros/mp_updatevarlength.sas b/001_macros/mp_updatevarlength.sas similarity index 100% rename from 005_macros/mp_updatevarlength.sas rename to 001_macros/mp_updatevarlength.sas diff --git a/009_macros/mp_validatecol.sas b/001_macros/mp_validatecol.sas similarity index 100% rename from 009_macros/mp_validatecol.sas rename to 001_macros/mp_validatecol.sas diff --git a/005_macros/mp_wait4file.sas b/001_macros/mp_wait4file.sas similarity index 100% rename from 005_macros/mp_wait4file.sas rename to 001_macros/mp_wait4file.sas diff --git a/005_macros/mp_webin.sas b/001_macros/mp_webin.sas similarity index 100% rename from 005_macros/mp_webin.sas rename to 001_macros/mp_webin.sas diff --git a/005_macros/mp_zip.sas b/001_macros/mp_zip.sas similarity index 100% rename from 005_macros/mp_zip.sas rename to 001_macros/mp_zip.sas diff --git a/004_macros/mddl_dc_difftable.sas b/002_macros/mddl_dc_difftable.sas similarity index 100% rename from 004_macros/mddl_dc_difftable.sas rename to 002_macros/mddl_dc_difftable.sas diff --git a/004_macros/mddl_dc_filterdetail.sas b/002_macros/mddl_dc_filterdetail.sas similarity index 100% rename from 004_macros/mddl_dc_filterdetail.sas rename to 002_macros/mddl_dc_filterdetail.sas diff --git a/004_macros/mddl_dc_filtersummary.sas b/002_macros/mddl_dc_filtersummary.sas similarity index 100% rename from 004_macros/mddl_dc_filtersummary.sas rename to 002_macros/mddl_dc_filtersummary.sas diff --git a/004_macros/mddl_dc_locktable.sas b/002_macros/mddl_dc_locktable.sas similarity index 100% rename from 004_macros/mddl_dc_locktable.sas rename to 002_macros/mddl_dc_locktable.sas diff --git a/004_macros/mddl_dc_maxkeytable.sas b/002_macros/mddl_dc_maxkeytable.sas similarity index 100% rename from 004_macros/mddl_dc_maxkeytable.sas rename to 002_macros/mddl_dc_maxkeytable.sas diff --git a/004_macros/mddl_sas_cntlout.sas b/002_macros/mddl_sas_cntlout.sas similarity index 100% rename from 004_macros/mddl_sas_cntlout.sas rename to 002_macros/mddl_sas_cntlout.sas diff --git a/002_macros/mm_assigndirectlib.sas b/002_macros/mm_assigndirectlib.sas deleted file mode 100644 index 8d118db..0000000 --- a/002_macros/mm_assigndirectlib.sas +++ /dev/null @@ -1,461 +0,0 @@ -/*** HELP START ***//** - @file - @brief Assigns library directly using details from metadata - @details Queries metadata to get the libname definition then allocates the - library directly (ie, not using the META engine). - usage: - - %mm_assignDirectLib(MyLib); - data x; set mylib.sometable; run; - - %mm_assignDirectLib(MyDB,open_passthrough=MyAlias); - create table MyTable as - select * from connection to MyAlias( select * from DBTable); - disconnect from MyAlias; - quit; - -

SAS Macros

- @li mf_getengine.sas - @li mp_abort.sas - - @param [in] libref the libref (not name) of the metadata library - @param [in] open_passthrough= () Provide an alias to produce the CONNECT TO - statement for the relevant external database - @param [in] sql_options= () Add any options to add to proc sql statement, - eg outobs= (only valid for pass through) - @param [in] mDebug= (0) set to 1 to show debug messages in the log - @param [in] mAbort= (0) set to 1 to call %mp_abort(). - - @returns libname statement - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_assigndirectlib( - libref - ,open_passthrough= - ,sql_options= - ,mDebug=0 - ,mAbort=0 -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_assigndirectlib.sas; -%&mD.put _local_; - -%if &mAbort=1 %then %let mAbort=; -%else %let mAbort=%str(*); - -%&mD.put NOTE: Creating direct (non META) connection to &libref library; - -%local cur_engine; -%let cur_engine=%mf_getengine(&libref); -%if &cur_engine ne META and &cur_engine ne %then %do; - %put NOTE: &libref already has a direct (&cur_engine) libname connection; - %return; -%end; -%else %if %upcase(&libref)=WORK %then %do; - %put NOTE: We already have a direct connection to WORK :-) ; - %return; -%end; - -/* need to determine the library ENGINE first */ -%local engine; -data _null_; - length lib_uri engine $256; - call missing (of _all_); - /* get URI for the particular library */ - rc1=metadata_getnobj("omsobj:SASLibrary?@Libref ='&libref'",1,lib_uri); - /* get the Engine attribute of the previous object */ - rc2=metadata_getattr(lib_uri,'Engine',engine); - putlog "mm_assigndirectlib for &libref:" rc1= lib_uri= rc2= engine=; - call symputx("liburi",lib_uri,'l'); - call symputx("engine",engine,'l'); -run; - -/* now obtain engine specific connection details */ -%if &engine=BASE %then %do; - %&mD.put NOTE: Retrieving BASE library path; - data _null_; - length up_uri $256 path cat_path $1024; - retain cat_path; - call missing (of _all_); - /* get all the filepaths of the UsingPackages association */ - i=1; - rc3=metadata_getnasn("&liburi",'UsingPackages',i,up_uri); - do while (rc3>0); - /* get the DirectoryName attribute of the previous object */ - rc4=metadata_getattr(up_uri,'DirectoryName',path); - if i=1 then path = '("'!!trim(path)!!'" '; - else path =' "'!!trim(path)!!'" '; - cat_path = trim(cat_path) !! " " !! trim(path) ; - i+1; - rc3=metadata_getnasn("&liburi",'UsingPackages',i,up_uri); - end; - cat_path = trim(cat_path) !! ")"; - &mD.putlog "NOTE: Getting physical path for &libref library"; - &mD.putlog rc3= up_uri= rc4= cat_path= path=; - &mD.putlog "NOTE: Libname cmd will be:"; - &mD.putlog "libname &libref" cat_path; - call symputx("filepath",cat_path,'l'); - run; - - %if %sysevalf(&sysver<9.4) %then %do; - libname &libref &filepath; - %end; - %else %do; - /* apply the new filelocks option to cater for temporary locks */ - libname &libref &filepath filelockwait=5; - %end; - -%end; -%else %if &engine=REMOTE %then %do; - data x; - length rcCon rcProp rc k 3 uriCon uriProp PropertyValue PropertyName - Delimiter $256 properties $2048; - retain properties; - rcCon = metadata_getnasn("&liburi", "LibraryConnection", 1, uriCon); - - rcProp = metadata_getnasn(uriCon, "Properties", 1, uriProp); - - k = 1; - rcProp = metadata_getnasn(uriCon, "Properties", k, uriProp); - do while (rcProp > 0); - rc = metadata_getattr(uriProp , "DefaultValue",PropertyValue); - rc = metadata_getattr(uriProp , "PropertyName",PropertyName); - rc = metadata_getattr(uriProp , "Delimiter",Delimiter); - properties = trim(properties) !! " " !! trim(PropertyName) - !! trim(Delimiter) !! trim(PropertyValue); - output; - k+1; - rcProp = metadata_getnasn(uriCon, "Properties", k, uriProp); - end; - %&mD.put NOTE: Getting properties for REMOTE SHARE &libref library; - &mD.put _all_; - %&mD.put NOTE: Libname cmd will be:; - %&mD.put libname &libref &engine &properties slibref=&libref; - call symputx ("properties",trim(properties),'l'); - run; - - libname &libref &engine &properties slibref=&libref; - -%end; - -%else %if &engine=OLEDB %then %do; - %&mD.put NOTE: Retrieving OLEDB connection details; - data _null_; - length domain datasource provider properties schema - connx_uri domain_uri conprop_uri lib_uri schema_uri value $256.; - call missing (of _all_); - /* get source connection ID */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri); - /* get connection domain */ - rc1=metadata_getnasn(connx_uri,'Domain',1,domain_uri); - rc2=metadata_getattr(domain_uri,'Name',domain); - &mD.putlog / 'NOTE: ' // 'NOTE- connection id: ' connx_uri ; - &mD.putlog 'NOTE- domain: ' domain; - /* get DSN and PROVIDER from connection properties */ - i=0; - do until (rc<0); - i+1; - rc=metadata_getnasn(connx_uri,'Properties',i,conprop_uri); - rc2=metadata_getattr(conprop_uri,'Name',value); - if value='Connection.OLE.Property.DATASOURCE.Name.xmlKey.txt' then do; - rc3=metadata_getattr(conprop_uri,'DefaultValue',datasource); - end; - else if value='Connection.OLE.Property.PROVIDER.Name.xmlKey.txt' then do; - rc4=metadata_getattr(conprop_uri,'DefaultValue',provider); - end; - else if value='Connection.OLE.Property.PROPERTIES.Name.xmlKey.txt' then - do; - rc5=metadata_getattr(conprop_uri,'DefaultValue',properties); - end; - end; - &mD.putlog 'NOTE- dsn/provider/properties: ' / - datasource provider properties; - &mD.putlog 'NOTE- schema: ' schema // 'NOTE-'; - - /* get SCHEMA */ - rc6=metadata_getnasn("&liburi",'UsingPackages',1,lib_uri); - rc7=metadata_getattr(lib_uri,'SchemaName',schema); - call symputx('SQL_domain',domain,'l'); - call symputx('SQL_dsn',datasource,'l'); - call symputx('SQL_provider',provider,'l'); - call symputx('SQL_properties',properties,'l'); - call symputx('SQL_schema',schema,'l'); - run; - - %if %length(&open_passthrough)>0 %then %do; - proc sql &sql_options; - connect to OLEDB as &open_passthrough(INSERT_SQL=YES - /* need additional properties to make this work */ - properties=('Integrated Security'=SSPI - 'Persist Security Info'=True - %sysfunc(compress(%str(&SQL_properties),%str(()))) - ) - DATASOURCE=&sql_dsn PROMPT=NO - PROVIDER=&sql_provider SCHEMA=&sql_schema CONNECTION = GLOBAL); - %end; - %else %do; - LIBNAME &libref OLEDB PROPERTIES=&sql_properties - DATASOURCE=&sql_dsn PROVIDER=&sql_provider SCHEMA=&sql_schema - %if %length(&sql_domain)>0 %then %do; - authdomain="&sql_domain" - %end; - connection=shared; - %end; -%end; -%else %if &engine=ODBC %then %do; - &mD.%put NOTE: Retrieving ODBC connection details; - data _null_; - length connx_uri conprop_uri value datasource up_uri schema $256.; - call missing (of _all_); - /* get source connection ID */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri); - /* get connection properties */ - i=0; - do until (rc2<0); - i+1; - rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri); - rc3=metadata_getattr(conprop_uri,'Name',value); - if value='Connection.ODBC.Property.DATASRC.Name.xmlKey.txt' then do; - rc4=metadata_getattr(conprop_uri,'DefaultValue',datasource); - rc2=-1; - end; - end; - /* get SCHEMA */ - rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri); - rc7=metadata_getattr(up_uri,'SchemaName',schema); - &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= datasource= - rc6= up_uri= rc7= schema=; - - call symputx('SQL_schema',schema,'l'); - call symputx('SQL_dsn',datasource,'l'); - run; - - %if %length(&open_passthrough)>0 %then %do; - proc sql &sql_options; - connect to ODBC as &open_passthrough - (INSERT_SQL=YES DATASRC=&sql_dsn. CONNECTION=global); - %end; - %else %do; - libname &libref ODBC DATASRC=&sql_dsn SCHEMA=&sql_schema; - %end; -%end; -%else %if &engine=POSTGRES %then %do; - %put NOTE: Obtaining POSTGRES library details; - data _null_; - length database ignore_read_only_columns direct_exe preserve_col_names - preserve_tab_names server schema authdomain user password - prop name value uri urisrc $256.; - call missing (of _all_); - /* get database value */ - prop='Connection.DBMS.Property.DB.Name.xmlKey.txt'; - rc=metadata_getprop("&liburi",prop,database,""); - if database^='' then database='database='!!quote(trim(database)); - call symputx('database',database,'l'); - - /* get IGNORE_READ_ONLY_COLUMNS value */ - prop='Library.DBMS.Property.DBIROC.Name.xmlKey.txt'; - rc=metadata_getprop("&liburi",prop,ignore_read_only_columns,""); - if ignore_read_only_columns^='' then ignore_read_only_columns= - 'ignore_read_only_columns='!!ignore_read_only_columns; - call symputx('ignore_read_only_columns',ignore_read_only_columns,'l'); - - /* get DIRECT_EXE value */ - prop='Library.DBMS.Property.DirectExe.Name.xmlKey.txt'; - rc=metadata_getprop("&liburi",prop,direct_exe,""); - if direct_exe^='' then direct_exe='direct_exe='!!direct_exe; - call symputx('direct_exe',direct_exe,'l'); - - /* get PRESERVE_COL_NAMES value */ - prop='Library.DBMS.Property.PreserveColNames.Name.xmlKey.txt'; - rc=metadata_getprop("&liburi",prop,preserve_col_names,""); - if preserve_col_names^='' then preserve_col_names= - 'preserve_col_names='!!preserve_col_names; - call symputx('preserve_col_names',preserve_col_names,'l'); - - /* get PRESERVE_TAB_NAMES value */ - /* be careful with PRESERVE_TAB_NAMES=YES - it will mean your table will - become case sensitive!! */ - prop='Library.DBMS.Property.PreserveTabNames.Name.xmlKey.txt'; - rc=metadata_getprop("&liburi",prop,preserve_tab_names,""); - if preserve_tab_names^='' then preserve_tab_names= - 'preserve_tab_names='!!preserve_tab_names; - call symputx('preserve_tab_names',preserve_tab_names,'l'); - - /* get SERVER value */ - if metadata_getnasn("&liburi","LibraryConnection",1,uri)>0 then do; - prop='Connection.DBMS.Property.SERVER.Name.xmlKey.txt'; - rc=metadata_getprop(uri,prop,server,""); - end; - if server^='' then server='server='!!quote(cats(server)); - call symputx('server',server,'l'); - - /* get SCHEMA value */ - if metadata_getnasn("&liburi","UsingPackages",1,uri)>0 then do; - rc=metadata_getattr(uri,"SchemaName",schema); - end; - if schema^='' then schema='schema='!!schema; - call symputx('schema',schema,'l'); - - /* get AUTHDOMAIN value */ - /* this is only useful if the user account contains that auth domain - if metadata_getnasn("&liburi","DefaultLogin",1,uri)>0 then do; - rc=metadata_getnasn(uri,"Domain",1,urisrc); - rc=metadata_getattr(urisrc,"Name",authdomain); - end; - if authdomain^='' then authdomain='authdomain='!!quote(trim(authdomain)); - */ - call symputx('authdomain',authdomain,'l'); - - /* get user & pass */ - if authdomain='' & metadata_getnasn("&liburi","DefaultLogin",1,uri)>0 then - do; - rc=metadata_getattr(uri,"UserID",user); - rc=metadata_getattr(uri,"Password",password); - end; - if user^='' then do; - user='user='!!quote(trim(user)); - password='password='!!quote(trim(password)); - end; - call symputx('user',user,'l'); - call symputx('password',password,'l'); - - &md.put _all_; - run; - - %if %length(&open_passthrough)>0 %then %do; - %put %str(WARN)ING: Passthrough option for postgres not yet supported; - %return; - %end; - %else %do; - %if &mdebug=1 %then %do; - %put NOTE: Executing the following:/; - %put NOTE- libname &libref POSTGRES &database &ignore_read_only_columns; - %put NOTE- &direct_exe &preserve_col_names &preserve_tab_names; - %put NOTE- &server &schema &authdomain &user &password //; - %end; - libname &libref POSTGRES &database &ignore_read_only_columns &direct_exe - &preserve_col_names &preserve_tab_names &server &schema &authdomain - &user &password; - %end; -%end; -%else %if &engine=ORACLE %then %do; - %put NOTE: Obtaining &engine library details; - data _null_; - length assocuri1 assocuri2 assocuri3 authdomain path schema $256; - call missing (of _all_); - - /* get auth domain */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri1); - rc=metadata_getnasn(assocuri1,'Domain',1,assocuri2); - rc=metadata_getattr(assocuri2,"Name",authdomain); - call symputx('authdomain',authdomain,'l'); - - /* path */ - rc=metadata_getprop(assocuri1, - 'Connection.Oracle.Property.PATH.Name.xmlKey.txt',path); - call symputx('path',path,'l'); - - /* schema */ - rc=metadata_getnasn("&liburi",'UsingPackages',1,assocuri3); - rc=metadata_getattr(assocuri3,'SchemaName',schema); - call symputx('schema',schema,'l'); - run; - %put NOTE: Executing the following:/; %put NOTE-; - %put NOTE- libname &libref ORACLE path=&path schema=&schema; - %put NOTE- authdomain=&authdomain; - %put NOTE-; - libname &libref ORACLE path=&path schema=&schema authdomain=&authdomain; -%end; -%else %if &engine=SQLSVR %then %do; - %put NOTE: Obtaining &engine library details; - data _null; - length assocuri1 assocuri2 assocuri3 authdomain path schema userid - passwd $256; - call missing (of _all_); - - rc=metadata_getnasn("&liburi",'DefaultLogin',1,assocuri1); - rc=metadata_getattr(assocuri1,"UserID",userid); - rc=metadata_getattr(assocuri1,"Password",passwd); - call symputx('user',userid,'l'); - call symputx('pass',passwd,'l'); - - /* path */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2); - rc=metadata_getprop(assocuri2, - 'Connection.SQL.Property.Datasrc.Name.xmlKey.txt',path); - call symputx('path',path,'l'); - - /* schema */ - rc=metadata_getnasn("&liburi",'UsingPackages',1,assocuri3); - rc=metadata_getattr(assocuri3,'SchemaName',schema); - call symputx('schema',schema,'l'); - run; - - %put NOTE: Executing the following:/; %put NOTE-; - %put NOTE- libname &libref SQLSVR datasrc=&path schema=&schema ; - %put NOTE- user="&user" pass="XXX"; - %put NOTE-; - - libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass"; -%end; -%else %if &engine=TERADATA %then %do; - %put NOTE: Obtaining &engine library details; - data _null; - length assocuri1 assocuri2 assocuri3 authdomain path schema userid - passwd $256; - call missing (of _all_); - - /* get auth domain */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri1); - rc=metadata_getnasn(assocuri1,'Domain',1,assocuri2); - rc=metadata_getattr(assocuri2,"Name",authdomain); - call symputx('authdomain',authdomain,'l'); - - /* - rc=metadata_getnasn("&liburi",'DefaultLogin',1,assocuri1); - rc=metadata_getattr(assocuri1,"UserID",userid); - rc=metadata_getattr(assocuri1,"Password",passwd); - call symputx('user',userid,'l'); - call symputx('pass',passwd,'l'); - */ - - /* path */ - rc=metadata_getnasn("&liburi",'LibraryConnection',1,assocuri2); - rc=metadata_getprop(assocuri2, - 'Connection.Teradata.Property.SERVER.Name.xmlKey.txt',path); - call symputx('path',path,'l'); - - /* schema */ - rc=metadata_getnasn("&liburi",'UsingPackages',1,assocuri3); - rc=metadata_getattr(assocuri3,'SchemaName',schema); - call symputx('schema',schema,'l'); - run; - - %put NOTE: Executing the following:/; %put NOTE-; - %put NOTE- libname &libref TERADATA server="&path" schema=&schema ; - %put NOTe- authdomain=&authdomain; - %put NOTE-; - - libname &libref TERADATA server="&path" schema=&schema authdomain=&authdomain; -%end; -%else %if &engine= %then %do; - %put NOTE: Libref &libref is not registered in metadata; - %&mAbort.mp_abort( - msg=%str(ERR)OR: Libref &libref is not registered in metadata - ,mac=mm_assigndirectlib.sas); - %return; -%end; -%else %do; - %put %str(WARN)ING: Engine &engine is currently unsupported; - %put %str(WARN)ING- Please contact your support team.; - %return; -%end; - -%mend mm_assigndirectlib; diff --git a/002_macros/mm_createstp.sas b/002_macros/mm_createstp.sas deleted file mode 100644 index d5a883d..0000000 --- a/002_macros/mm_createstp.sas +++ /dev/null @@ -1,403 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a type 1 Stored Process (9.2 compatible) - @details This macro creates a Type 1 stored process, and also the necessary - PromptGroup / File / TextStore objects. It requires the location (or uri) - for the App Server / Directory / Folder (Tree) objects. - To upgrade this macro to work with type 2 (which can embed SAS code - and is compabitible with SAS from 9.3 onwards) then the UsageVersion should - change to 2000000 and the TextStore object updated. The ComputeServer - reference will also be to ServerContext rather than LogicalServer. - - This macro is idempotent - if you run it twice, it will only create an STP - once. - - Usage (type 1 STP): - - %mm_createstp(stpname=MyNewSTP - ,filename=mySpecialProgram.sas - ,directory=SASEnvironment/SASCode/STPs - ,tree=/User Folders/sasdemo - ,outds=work.uris) - - If you wish to remove the new STP you can do so by running: - - data _null_; - set work.uris; - rc1 = METADATA_DELOBJ(texturi); - rc2 = METADATA_DELOBJ(prompturi); - rc3 = METADATA_DELOBJ(fileuri); - rc4 = METADATA_DELOBJ(stpuri); - putlog (_all_)(=); - run; - - Usage (type 2 STP): - - %mm_createstp(stpname=MyNewType2STP - ,filename=mySpecialProgram.sas - ,directory=SASEnvironment/SASCode/STPs - ,tree=/User Folders/sasdemo - ,Server=SASApp - ,stptype=2) - - @param [in] stpname= (SASjs Default STP) Stored Process name. - Avoid spaces - testing has shown that - the check to avoid creating multiple STPs in the same folder with the same - name does not work when the name contains spaces. - @param [in] stpdesc= Stored Process description (optional) - @param [in] filename= the name of the .sas program to run - @param [in] directory= (SASEnvironment/sascode) - The directory uri or the actual path to the sas program (no trailing slash). - If more than uri is found with that path, then the first one will be used. - @param [in] tree= The metadata folder uri, or the metadata path, in which to - create the STP. - @param [in] server= (SASApp) The server which will run the STP. - Server name or uri is fine. - @param [out] outds= (work.mm_createstp) - The two level name of the output dataset. Will contain all the meta uris. - @param [in] mDebug= set to 1 to show debug messages in the log - @param [in] stptype= Default is 1 (STP code saved on filesystem). Set to 2 if - source code is to be saved in metadata (9.3 and above feature). - @param [in] minify= set to YES to strip comments / blank lines etc - @param [in] frefin= (mm_in) fileref to use (enables change if there is - a conflict). - The filerefs are left open, to enable inspection after running the - macro (or importing into an xmlmap if needed). - @param [out] frefout= (mm_out) fileref to use (enables change if there is - a conflict) - @param [in] repo= ServerContext is tied to a repo, if you are not using the - foundation repo then select a different one here - - @returns outds dataset containing the following columns: - - stpuri - - prompturi - - fileuri - - texturi - -

SAS Macros

- @li mf_nobs.sas - @li mf_verifymacvars.sas - @li mm_getdirectories.sas - @li mm_updatestpsourcecode.sas - @li mm_getservercontexts.sas - @li mp_abort.sas - @li mp_dropmembers.sas - -

Related Macros

- @li mm_createwebservice.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createstp( - stpname=SASjs Default STP - ,stpdesc=This stp was created automatically by the mm_createstp macro - ,filename=mm_createstp.sas - ,directory=SASEnvironment/SASCode - ,tree=/User Folders/sasdemo - ,package=false - ,streaming=true - ,outds=work.mm_createstp - ,mDebug=0 - ,server=SASApp - ,stptype=1 - ,minify=NO - ,frefin=mm_in - ,frefout=mm_out -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_CreateSTP.sas; -%&mD.put _local_; - -%mp_abort( - iftrue=(%mf_verifymacvars(stpname filename directory tree)=0) - ,mac=&sysmacroname - ,msg=%str(Empty inputs: stpname filename directory tree) -) - -%mp_dropmembers(%scan(&outds,2,.)) - -/** - * check tree exists - */ -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree","Folder",type,uri); - call symputx('foldertype',type,'l'); - call symputx('treeuri',uri,'l'); -run; -%if &foldertype ne Tree %then %do; - %put %str(WARN)ING: Tree &tree does not exist!; - %return; -%end; - -/** - * Check STP does not exist already - */ -%local cmtype; -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree/&stpname",'StoredProcess',type,uri); - call symputx('cmtype',type,'l'); - call symputx('stpuri',uri,'l'); -run; -%if &cmtype = ClassifierMap %then %do; - %put %str(WARN)ING: Stored Process &stpname already exists in &tree!; - %return; -%end; - -/** - * Check that the physical file exists - */ -%if %sysfunc(fileexist(&directory/&filename)) ne 1 %then %do; - %put %str(WARN)ING: FILE *&directory/&filename* NOT FOUND!; - %return; -%end; - -%if &stptype=1 %then %do; - /* type 1 STP - where code is stored on filesystem */ - %if %sysevalf(&sysver lt 9.2) %then %do; - %put %str(WARN)ING: Version 9.2 or later required; - %return; - %end; - - /* check directory object (where 9.2 source code reference is stored) */ - data _null_; - length id $20 dirtype $256; - rc=metadata_resolve("&directory",dirtype,id); - call symputx('checkdirtype',dirtype,'l'); - run; - - %if &checkdirtype ne Directory %then %do; - %mm_getdirectories(path=&directory,outds=&outds ,mDebug=&mDebug) - %if %mf_nobs(&outds)=0 or %sysfunc(exist(&outds))=0 %then %do; - %put %str(WARN)ING: The directory object does not exist for &directory; - %return; - %end; - %end; - %else %do; - data &outds; - directoryuri="&directory"; - run; - %end; - - data &outds (keep=stpuri prompturi fileuri texturi); - length stpuri prompturi fileuri texturi serveruri $256 ; - if _n_=1 then call missing (of _all_); - set &outds; - - /* final checks on uris */ - length id $20 type $256; - __rc=metadata_resolve("&treeuri",type,id); - if type ne 'Tree' then do; - putlog "%str(WARN)ING: Invalid tree URI: &treeuri"; - stopme=1; - end; - __rc=metadata_resolve(directoryuri,type,id); - if type ne 'Directory' then do; - putlog "%str(WARN)ING: Invalid directory URI: " directoryuri; - stopme=1; - end; - - /* get server info */ - __rc=metadata_resolve("&server",type,serveruri); - if type ne 'LogicalServer' then do; - __rc=metadata_getnobj("omsobj:LogicalServer?@Name='&server'",1,serveruri); - if serveruri='' then do; - putlog "%str(WARN)ING: Invalid server: &server"; - stopme=1; - end; - end; - - if stopme=1 then do; - putlog (_all_)(=); - stop; - end; - - /* create empty prompt */ - rc1=METADATA_NEWOBJ('PromptGroup',prompturi,'Parameters'); - rc2=METADATA_SETATTR(prompturi, 'UsageVersion', '1000000'); - rc3=METADATA_SETATTR(prompturi, 'GroupType','2'); - rc4=METADATA_SETATTR(prompturi, 'Name','Parameters'); - rc5=METADATA_SETATTR(prompturi, 'PublicType','Embedded:PromptGroup'); - GroupInfo= - ""; - rc6 = METADATA_SETATTR(prompturi, 'GroupInfo',groupinfo); - - if sum(of rc1-rc6) ne 0 then do; - putlog "%str(WARN)ING: Issue creating prompt."; - if prompturi ne . then do; - putlog ' Removing orphan: ' prompturi; - rc = METADATA_DELOBJ(prompturi); - put rc=; - end; - stop; - end; - - /* create a file uri */ - rc7=METADATA_NEWOBJ('File',fileuri,'SP Source File'); - rc8=METADATA_SETATTR(fileuri, 'FileName',"&filename"); - rc9=METADATA_SETATTR(fileuri, 'IsARelativeName','1'); - rc10=METADATA_SETASSN(fileuri, 'Directories','MODIFY',directoryuri); - if sum(of rc7-rc10) ne 0 then do; - putlog "%str(WARN)ING: Issue creating file."; - if fileuri ne . then do; - putlog ' Removing orphans:' prompturi fileuri; - rc = METADATA_DELOBJ(prompturi); - rc = METADATA_DELOBJ(fileuri); - put (_all_)(=); - end; - stop; - end; - - /* create a TextStore object */ - rc11= METADATA_NEWOBJ('TextStore',texturi,'Stored Process'); - rc12= METADATA_SETATTR(texturi, 'TextRole','StoredProcessConfiguration'); - rc13= METADATA_SETATTR(texturi, 'TextType','XML'); - storedtext='' - !!"" - !!""; - rc14= METADATA_SETATTR(texturi, 'StoredText',storedtext); - if sum(of rc11-rc14) ne 0 then do; - putlog "%str(WARN)ING: Issue creating TextStore."; - if texturi ne . then do; - putlog ' Removing orphans: ' prompturi fileuri texturi; - rc = METADATA_DELOBJ(prompturi); - rc = METADATA_DELOBJ(fileuri); - rc = METADATA_DELOBJ(texturi); - put (_all_)(=); - end; - stop; - end; - - /* create meta obj */ - rc15= METADATA_NEWOBJ('ClassifierMap',stpuri,"&stpname"); - rc16= METADATA_SETASSN(stpuri, 'Trees','MODIFY',treeuri); - rc17= METADATA_SETASSN(stpuri, 'ComputeLocations','MODIFY',serveruri); - rc18= METADATA_SETASSN(stpuri, 'SourceCode','MODIFY',fileuri); - rc19= METADATA_SETASSN(stpuri, 'Prompts','MODIFY',prompturi); - rc20= METADATA_SETASSN(stpuri, 'Notes','MODIFY',texturi); - rc21= METADATA_SETATTR(stpuri, 'PublicType', 'StoredProcess'); - rc22= METADATA_SETATTR(stpuri, 'TransformRole', 'StoredProcess'); - rc23= METADATA_SETATTR(stpuri, 'UsageVersion', '1000000'); - rc24= METADATA_SETATTR(stpuri, 'Desc', "&stpdesc"); - - /* tidy up if err */ - if sum(of rc15-rc24) ne 0 then do; - putlog "%str(WARN)ING: Issue creating STP."; - if stpuri ne . then do; - putlog ' Removing orphans: ' prompturi fileuri texturi stpuri; - rc = METADATA_DELOBJ(prompturi); - rc = METADATA_DELOBJ(fileuri); - rc = METADATA_DELOBJ(texturi); - rc = METADATA_DELOBJ(stpuri); - put (_all_)(=); - end; - end; - else do; - fullpath=cats('_program=',treepath,"/&stpname"); - putlog "NOTE: Stored Process Created!"; - putlog "NOTE- "; putlog "NOTE-"; putlog "NOTE-" fullpath; - putlog "NOTE- "; putlog "NOTE-"; - end; - output; - stop; - run; -%end; -%else %if &stptype=2 %then %do; - /* type 2 stp - code is stored in metadata */ - %if %sysevalf(&sysver lt 9.3) %then %do; - %put %str(WARN)ING: SAS version 9.3 or later required to create type2 STPs; - %return; - %end; - /* check we have the correct ServerContext */ - %mm_getservercontexts(outds=contexts) - %local serveruri; %let serveruri=NOTFOUND; - data _null_; - set contexts; - where upcase(servername)="%upcase(&server)"; - call symputx('serveruri',serveruri); - run; - %if &serveruri=NOTFOUND %then %do; - %put %str(WARN)ING: ServerContext *&server* not found!; - %return; - %end; - - /** - * First, create a Hello World type 2 stored process - */ - filename &frefin temp; - data _null_; - file &frefin; - treeuri=quote(symget('treeuri')); - serveruri=quote(symget('serveruri')); - stpdesc=quote(symget('stpdesc')); - stpname=quote(symget('stpname')); - - put "$METAREPOSITORY "/ - ''/ - " "/ - " "/ - " "/ - " "/ - ' '/ - ' ' / - " "/ - " "/ - ' '/ - " "/ - ""/ - "SAS"/ - "268435456"; - run; - - filename &frefout temp; - - proc metadata in= &frefin out=&frefout ; - run; - - %if &mdebug=1 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile &frefout lrecl=1048576; - input; - put _infile_; - run; - %end; - - /** - * Next, add the source code - */ - %mm_updatestpsourcecode(stp=&tree/&stpname - ,stpcode="&directory/&filename" - ,mdebug=&mdebug - ,minify=&minify) - - -%end; -%else %do; - %put %str(WARN)ING: STPTYPE=*&stptype* not recognised!; -%end; - -%mend mm_createstp; diff --git a/002_macros/mm_getgroups.sas b/002_macros/mm_getgroups.sas deleted file mode 100644 index 42eef58..0000000 --- a/002_macros/mm_getgroups.sas +++ /dev/null @@ -1,95 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates dataset with all groups or just those for a particular user - @details Provide a metadata user to get groups for just that user, or leave - blank to return all groups. - Usage: - - - all groups: `%mm_getGroups()` - - - all groups for a particular user: `%mm_getgroups(user=&sysuserid)` - - @param [in] user= the metadata user to return groups for. Leave blank for all - groups. - @param [in] repo= the metadata repository that contains the user/group - information - @param [in] mDebug= set to 1 to show debug messages in the log - @param [out] outds= the dataset to create that contains the list of groups - - @returns outds dataset containing all groups in a column named "metagroup" - - groupuri - - groupname - - groupdesc - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getGroups( - user= - ,outds=work.mm_getGroups - ,repo=foundation - ,mDebug=0 -)/*/STORE SOURCE*/; - -%local mD oldrepo; -%let oldrepo=%sysfunc(getoption(metarepository)); -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_getGroups.sas; -%&mD.put _local_; - -/* on some sites, user / group info is in a different metadata repo to the - default */ -%if &oldrepo ne &repo %then %do; - options metarepository=&repo; -%end; - -%if %length(&user)=0 %then %do; - data &outds (keep=groupuri groupname groupdesc); - length groupuri groupname groupdesc group_or_role $256; - call missing(of _all_); - i+1; - do while - (metadata_getnobj("omsobj:IdentityGroup?@Id contains '.'",i,groupuri)>0); - rc=metadata_getattr(groupuri, "Name", groupname); - rc=metadata_getattr(groupuri, "Desc", groupdesc); - rc=metadata_getattr(groupuri,"PublicType",group_or_role); - if Group_or_Role = 'UserGroup' then output; - i+1; - end; - run; -%end; -%else %do; - data &outds (keep=groupuri groupname groupdesc); - length uri groupuri groupname groupdesc group_or_role $256; - call missing(of _all_); - rc=metadata_getnobj("omsobj:Person?@Name='&user'",1,uri); - if rc<=0 then do; - putlog "%str(WARN)ING: rc=" rc "&user not found " - ", or there was an issue reading the repository."; - stop; - end; - a=1; - grpassn=metadata_getnasn(uri,"IdentityGroups",a,groupuri); - if grpassn in (-3,-4) then do; - putlog "%str(WARN)ING: No metadata groups found for &user"; - output; - end; - else do while (grpassn > 0); - rc=metadata_getattr(groupuri, "Name", groupname); - rc=metadata_getattr(groupuri, "Desc", groupdesc); - a+1; - rc=metadata_getattr(groupuri,"PublicType",group_or_role); - if Group_or_Role = 'UserGroup' then output; - grpassn=metadata_getnasn(uri,"IdentityGroups",a,groupuri); - end; - run; -%end; - -%if &oldrepo ne &repo %then %do; - options metarepository=&oldrepo; -%end; - -%mend mm_getGroups; diff --git a/001_macros/mcf_getfmttype.sas b/003_macros/mcf_getfmttype.sas similarity index 100% rename from 001_macros/mcf_getfmttype.sas rename to 003_macros/mcf_getfmttype.sas diff --git a/001_macros/mcf_init.sas b/003_macros/mcf_init.sas similarity index 100% rename from 001_macros/mcf_init.sas rename to 003_macros/mcf_init.sas diff --git a/001_macros/mcf_length.sas b/003_macros/mcf_length.sas similarity index 100% rename from 001_macros/mcf_length.sas rename to 003_macros/mcf_length.sas diff --git a/001_macros/mcf_string2file.sas b/003_macros/mcf_string2file.sas similarity index 100% rename from 001_macros/mcf_string2file.sas rename to 003_macros/mcf_string2file.sas diff --git a/003_macros/mfv_existfile.sas b/003_macros/mfv_existfile.sas deleted file mode 100644 index 8b25cb0..0000000 --- a/003_macros/mfv_existfile.sas +++ /dev/null @@ -1,52 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a file exists in SAS Drive - @details Returns 1 if the file exists, and 0 if it doesn't. Works by - attempting to assign a fileref with the filesrvc engine. If not found, the - syscc is automatically set to a non zero value - so in this case it is reset. - To avoid hiding issues, there is therefore a test at the start to ensure the - syscc is zero. - - Usage: - - %put %mfv_existfile(/does/exist.txt); - %put %mfv_existfile(/does/not/exist.txt); - - @param [in] filepath The full path to the file on SAS drive - (eg /Public/myfile.txt) - -

SAS Macros

- @li mf_abort.sas - @li mf_getuniquefileref.sas - -

Related Macros

- @li mfv_existfolder.sas - - @version 3.5 - @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/) -**//*** HELP END ***/ - -%macro mfv_existfile(filepath -)/*/STORE SOURCE*/; - - %mf_abort( - iftrue=(&syscc ne 0), - msg=Cannot enter mfv_existfile.sas with syscc=&syscc - ) - - %local fref rc path name; - %let fref=%mf_getuniquefileref(); - %let name=%scan(&filepath,-1,/); - %let path=%substr(&filepath,1,%length(&filepath)-%length(&name)-1); - - %if %sysfunc(filename(fref,,filesrvc,folderPath="&path" filename="&name"))=0 - %then %do; - %sysfunc(fexist(&fref)) - %let rc=%sysfunc(filename(fref)); - %end; - %else %do; - 0 - %let syscc=0; - %end; - -%mend mfv_existfile; diff --git a/003_macros/mfv_existfolder.sas b/003_macros/mfv_existfolder.sas deleted file mode 100644 index 8fb73fb..0000000 --- a/003_macros/mfv_existfolder.sas +++ /dev/null @@ -1,48 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a folder exists in SAS Drive - @details Returns 1 if the folder exists, and 0 if it doesn't. Works by - attempting to assign a fileref with the filesrvc engine. If not found, the - syscc is automatically set to a non zero value - so in this case it is reset. - To avoid hiding issues, there is therefore a test at the start to ensure the - syscc is zero. - - Usage: - - %put %mfv_existfolder(/does/exist); - %put %mfv_existfolder(/does/not/exist); - - @param [in] path The path to the folder on SAS drive - -

SAS Macros

- @li mf_abort.sas - @li mf_getuniquefileref.sas - -

Related Macros

- @li mfv_existfile.sas - - @version 3.5 - @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/) -**//*** HELP END ***/ - -%macro mfv_existfolder(path -)/*/STORE SOURCE*/; - - %mf_abort( - iftrue=(&syscc ne 0), - msg=Cannot enter mfv_existfolder.sas with syscc=&syscc - ) - - %local fref rc; - %let fref=%mf_getuniquefileref(); - - %if %sysfunc(filename(fref,,filesrvc,folderPath="&path"))=0 %then %do; - 1 - %let rc=%sysfunc(filename(fref)); - %end; - %else %do; - 0 - %let syscc=0; - %end; - -%mend mfv_existfolder; diff --git a/003_macros/mfv_existsashdat.sas b/003_macros/mfv_existsashdat.sas deleted file mode 100644 index 332fec6..0000000 --- a/003_macros/mfv_existsashdat.sas +++ /dev/null @@ -1,60 +0,0 @@ -/*** HELP START ***//** - @file mfv_existsashdat.sas - @brief Checks whether a CAS sashdat dataset exists in persistent storage. - @details Can be used in open code, eg as follows: - - %if %mfv_existsashdat(libds=casuser.sometable) %then %put yes it does!; - - The function uses `dosubl()` to run the `table.fileinfo` action, for the - specified library, filtering for `*.sashdat` tables. The results are stored - in a WORK table (&outprefix._&lib). If that table already exists, it is - queried instead, to avoid the dosubl() performance hit. - - To force a rescan, just use a new `&outprefix` value, or delete the table(s) - before running the function. - - @param [in] libds library.dataset - @param [out] outprefix= (work.mfv_existsashdat) - Used to store current HDATA tables to improve subsequent query performance. - This reference is a prefix and is converted to `&prefix._{libref}` - - @return output returns 1 or 0 - - @version 0.2 - @author Mathieu Blauw -**//*** HELP END ***/ - -%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat -); -%local rc dsid name lib ds; -%let lib=%upcase(%scan(&libds,1,'.')); -%let ds=%upcase(%scan(&libds,-1,'.')); - -/* if table does not exist, create it */ -%if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do; - %let rc=%sysfunc(dosubl(%nrstr( - /* Read in table list (once per &lib per session) */ - proc cas; - table.fileinfo result=source_list /caslib="&lib"; - val=findtable(source_list); - saveresult val dataout=&outprefix._&lib; - quit; - /* Only keep name, without file extension */ - data &outprefix._&lib; - set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name); - Name=upcase(scan(Name,1,'.')); - run; - ))); -%end; - -/* Scan table for hdat existence */ -%let dsid=%sysfunc(open(&outprefix._&lib(where=(name="&ds")))); -%syscall set(dsid); -%let rc = %sysfunc(fetch(&dsid)); -%let rc = %sysfunc(close(&dsid)); - -/* Return result */ -%if "%trim(&name)"="%trim(&ds)" %then 1; -%else 0; - -%mend mfv_existsashdat; diff --git a/003_macros/mm_adduser2group.sas b/003_macros/mm_adduser2group.sas deleted file mode 100644 index 9f28ac3..0000000 --- a/003_macros/mm_adduser2group.sas +++ /dev/null @@ -1,102 +0,0 @@ -/*** HELP START ***//** - @file mm_adduser2group.sas - @brief Adds a user to a group - @details Adds a user to a metadata group. The macro first checks whether the - user is in that group, and if not, the user is added. - - Note that the macro does not check inherited group memberships - it looks at - direct members only. - - Usage: - - %mm_adduser2group(user=sasdemo - ,group=someGroup) - - - @param [in] user= the user name (not displayname) - @param [in] group= the group to which to add the user - @param [in] mdebug= (0) set to 1 to show debug info in log - -

Related Files

- @li ms_adduser2group.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_adduser2group(user= - ,group= - ,mdebug=0 -); -/* first, check if user is in group already exists */ -%local check uuri guri; -%let check=ok; - -data _null_; - length uri type msg $256; - call missing(of _all_); - rc=metadata_getnobj("omsobj:Person?@Name='&user'",1,uri); - if rc<=0 then do; - msg="%str(WARN)ING: rc="!!cats(rc)!!" &user not found "!! - ", or there was an err reading the repository."; - call symputx('check',msg); - putlog msg; - stop; - end; - call symputx('uuri',scan(uri,2,'\')); - - rc=metadata_getnobj("omsobj:IdentityGroup?@Name='&group'",1,uri); - if rc<=0 then do; - msg="%str(WARN)ING: rc="!!cats(rc)!!" &group not found "!! - ", or there was an err reading the repository."; - call symputx('check',msg); - putlog msg; - stop; - end; - call symputx('guri',scan(uri,2,'\')); - - rc=metadata_getnobj("omsobj:Person?Person[@Name='&user'][IdentityGroups/*[@Name='&group']]",1,uri); - if rc=0 then do; - msg="%str(WARN)ING: rc="!!cats(rc)!!" &user already in &group"; - call symputx('check',msg); - stop; - end; - - if &mdebug ne 0 then put (_all_)(=); -run; - -/* stop if issues */ -%if %quote(&check) ne %quote(ok) %then %do; - %put ✓ - %return; -%end; - -%if &syscc ge 4 %then %do; - %put %str(WARN)ING: SYSCC=&syscc, exiting &sysmacroname; - %return; -%end; - - -filename __us2grp temp; - -proc metadata in= "$METAREPOSITORY - - - - SAS268435456" - out=__us2grp verbose; -run; - -%if &mdebug ne 0 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile __us2grp lrecl=32767; - input; - put _infile_; - run; -%end; - -filename __us2grp clear; - -%mend mm_adduser2group; diff --git a/003_macros/mm_assignlib.sas b/003_macros/mm_assignlib.sas deleted file mode 100644 index c33a252..0000000 --- a/003_macros/mm_assignlib.sas +++ /dev/null @@ -1,97 +0,0 @@ -/*** HELP START ***//** - @file - @brief Assigns a meta engine library using LIBREF - @details Queries metadata to get the library NAME which can then be used in - a libname statement with the meta engine. - - usage: - - %macro mp_abort(iftrue,mac,msg);%put &=msg;%mend; - - %mm_assignlib(SOMEREF) - -

SAS Macros

- @li mp_abort.sas - - @param [in] libref The libref (not name) of the metadata library - @param [in] mAbort= (HARD) If not assigned, HARD will call %mp_abort(), SOFT - will silently return - - @returns libname statement - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_assignlib( - libref - ,mAbort=HARD -)/*/STORE SOURCE*/; -%local mp_abort msg; -%let mp_abort=0; -%if %sysfunc(libref(&libref)) %then %do; - data _null_; - length liburi LibName msg $200; - call missing(of _all_); - nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi); - if nobj=1 then do; - rc=metadata_getattr(liburi,"Name",LibName); - /* now try and assign it */ - if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do; - putlog "&libref could not be assigned"; - putlog liburi=; - /** - * Fetch the system message for display in the abort modal. This is - * not always helpful though. One example, previously received: - * NOTE: Libref XX refers to the same library metadata as libref XX. - */ - msg=sysmsg(); - if msg=:'ERROR: Libref SAVE is not assigned.' then do; - msg=catx(" ", - "Could not assign %upcase(&libref).", - "Please check metadata permissions! Libname:",libname, - "Liburi:",liburi - ); - end; - else if msg="ERROR: User does not have appropriate authorization "!! - "level for library SAVE." - then do; - msg=catx(" ", - "ERROR: User does not have appropriate authorization level", - "for library %upcase(&libref), libname:",libname, - "Liburi:",liburi - ); - end; - call symputx('msg',msg,'l'); - if "&mabort"='HARD' then call symputx('mp_abort',1,'l'); - end; - else do; - put (_all_)(=); - call symputx('libname',libname,'L'); - call symputx('liburi',liburi,'L'); - end; - end; - else if nobj>1 then do; - if "&mabort"='HARD' then call symputx('mp_abort',1); - call symputx('msg',"More than one library with libref=&libref"); - end; - else do; - if "&mabort"='HARD' then call symputx('mp_abort',1); - call symputx('msg',"Library &libref not found in metadata"); - end; - run; - - %put NOTE: &msg; - -%end; -%else %do; - %put NOTE: Library &libref is already assigned; -%end; - -%mp_abort(iftrue= (&mp_abort=1) - ,mac=mm_assignlib.sas - ,msg=%superq(msg) -) - -%mend mm_assignlib; diff --git a/003_macros/mm_createapplication.sas b/003_macros/mm_createapplication.sas deleted file mode 100644 index 25cab62..0000000 --- a/003_macros/mm_createapplication.sas +++ /dev/null @@ -1,161 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create an Application object in a metadata folder - @details Application objects are useful for storing properties in metadata. - This macro is idempotent - it will not create an object with the same name - in the same location, twice. - - usage: - - %mm_createapplication(tree=/User Folders/sasdemo - ,name=MyApp - ,classidentifier=myAppSeries - ,params= name1=value1 name2=value2 emptyvalue= - ) - - @warning application components do not get deleted when removing the container - folder! be sure you have the administrative priviliges to remove this kind of - metadata from the SMC plugin (or be ready to do to so programmatically). - -

SAS Macros

- @li mp_abort.sas - @li mf_verifymacvars.sas - - @param [in] tree= The metadata folder uri, or the metadata path, in which to - create the object. This must exist. - @param [in] name= Application object name. Avoid spaces. - @param [in] ClassIdentifier= - The class of applications to which this app belongs - @param [in] params= - name=value pairs which will become public properties of the - application object. These are delimited using (newline character) - - @param [in] desc= Application description (optional). - Avoid ampersands as these - are illegal characters (unless they are escapted- eg &) - @param [in] version= version number of application - @param [in] frefin= fileref to use (enables change if there is a conflict). - The filerefs are left open, to enable inspection after running the - macro (or importing into an xmlmap if needed). - @param [out] frefout= fileref to use (enables change if there is a conflict) - @param [in] mDebug= set to 1 to show debug messages in the log - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createapplication( - tree=/User Folders/sasdemo - ,name=myApp - ,ClassIdentifier=mcore - ,desc=Created by mm_createapplication - ,params= param1=1 param2=blah - ,version= - ,frefin=mm_in - ,frefout=mm_out - ,mDebug=1 - ); - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0) - ,mac=&sysmacroname - ,msg=%str(Empty inputs: tree name) -) - -/** - * check tree exists - */ - -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree","Folder",type,uri); - call symputx('type',type,'l'); - call symputx('treeuri',uri,'l'); -run; - -%mp_abort( - iftrue= (&type ne Tree) - ,mac=mm_createapplication.sas - ,msg=Tree &tree does not exist! -) - -/** - * Check object does not exist already - */ -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree/&name","Application",type,uri); - call symputx('type',type,'l'); - putlog (_all_)(=); -run; - -%mp_abort( - iftrue= (&type = SoftwareComponent) - ,mac=mm_createapplication.sas - ,msg=Application &name already exists in &tree! -) - - -/** - * Now we can create the application - */ -filename &frefin temp; - -/* write header XML */ -data _null_; - file &frefin; - name=quote(symget('name')); - desc=quote(symget('desc')); - ClassIdentifier=quote(symget('ClassIdentifier')); - version=quote(symget('version')); - params=quote(symget('params')); - treeuri=quote(symget('treeuri')); - - put "$METAREPOSITORY "/ - '' / - ' ' / - ' ' / - ' ' / - ""/ - "SAS"/ - "268435456"; -run; - -filename &frefout temp; - -proc metadata in= &frefin out=&frefout verbose; -run; - -%if &mdebug=1 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile &frefout lrecl=1048576; - input; - put _infile_; - run; -%end; - -%put NOTE: Checking to ensure application (&name) was created; -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree/&name","Application",type,uri); - call symputx('apptype',type,'l'); - %if &mdebug=1 %then putlog (_all_)(=);; -run; -%if &apptype ne SoftwareComponent %then %do; - %put %str(ERR)OR: Could not find (&name) at (&tree)!!; - %return; -%end; -%else %put NOTE: Application (&name) successfully created in (&tree)!; - - -%mend mm_createapplication; diff --git a/003_macros/mm_createdataset.sas b/003_macros/mm_createdataset.sas deleted file mode 100644 index 695e01a..0000000 --- a/003_macros/mm_createdataset.sas +++ /dev/null @@ -1,91 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create an empty dataset from a metadata definition - @details This macro was built to support viewing empty tables in - https://datacontroller.io - - The table can be retrieved using LIBRARY.DATASET reference, or directly - using the metadata URI. - - The dataset is written to the WORK library. - - Usage: - - %mm_createdataset(libds=metlib.some_dataset) - - or - - %mm_createdataset(tableuri=G5X8AFW1.BE00015Y) - -

SAS Macros

- @li mm_getlibs.sas - @li mm_gettables.sas - @li mm_getcols.sas - - @param [in] libds= library.dataset metadata source. Note - table names in metadata - can be longer than 32 chars (just fyi, not an issue here) - @param [in] tableuri= Metadata URI of the table to be created - @param [out] outds= (work.mm_createdataset) The dataset to create. The table - name needs to be 32 chars or less as per SAS naming rules. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createdataset(libds=,tableuri=,outds=work.mm_createdataset,mDebug=0); -%local dbg errorcheck tempds1 tempds2 tempds3; -%if &mDebug=0 %then %let dbg=*; -%let errorcheck=1; - -%if %index(&libds,.)>0 %then %do; - /* get lib uri */ - data;run;%let tempds1=&syslast; - %mm_getlibs(outds=&tempds1) - data _null_; - set &tempds1; - if upcase(libraryref)="%upcase(%scan(&libds,1,.))"; - call symputx('liburi',LibraryId,'l'); - run; - /* get ds uri */ - data;run;%let tempds2=&syslast; - %mm_gettables(uri=&liburi,outds=&tempds2) - data _null_; - set &tempds2; - where upcase(tablename)="%upcase(%scan(&libds,2,.))"; - &dbg putlog tableuri=; - call symputx('tableuri',tableuri); - run; -%end; - -data;run; -%let tempds3=&syslast; -%mm_getcols(tableuri=&tableuri,outds=&tempds3) - -%if %mf_nobs(&tempds3)=0 %then %do; - %put &libds (&tableuri) has no columns defined!!; - data &outds; - run; - %return; -%end; - -data _null_; - set &tempds3 end=last; - if _n_=1 then call execute('data &outds;'); - length attrib $32767; - - if SAScolumntype='C' then type='$'; - attrib='attrib '!!cats(colname)!!' length='!!cats(type,SASColumnLength,'.'); - - if not missing(sasformat) then fmt=' format='!!cats(sasformat); - if not missing(sasinformat) then infmt=' informat='!!cats(sasinformat); - if not missing(coldesc) then desc=' label='!!quote(cats(coldesc)); - - attrib=trim(attrib)!!fmt!!infmt!!desc!!';'; - - call execute(attrib); - if last then call execute('call missing(of _all_);stop;run;'); -run; - -%mend mm_createdataset; diff --git a/003_macros/mm_createdocument.sas b/003_macros/mm_createdocument.sas deleted file mode 100644 index 9237a73..0000000 --- a/003_macros/mm_createdocument.sas +++ /dev/null @@ -1,128 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a Document object in a metadata folder - @details Document objects are useful for storing properties in metadata. - This macro is idempotent - it will not create an object with the same name - in the same location, twice. - Note - the filerefs are left open, to enable inspection after running the - macro (or importing into an xmlmap if needed). - - usage: - - %mm_createdocument(tree=/User Folders/sasdemo - ,name=MyNote) - -

SAS Macros

- @li mp_abort.sas - @li mf_verifymacvars.sas - - - @param [in] tree= The metadata folder uri, or the metadata path, in which to - create the document. This must exist. - @param [in] name= Document object name. Avoid spaces. - - @param [in] desc= Document description (optional) - @param [in] textrole= TextRole property (optional) - @param [in] frefin= fileref to use (enables change if there is a conflict) - @param [out] frefout= fileref to use (enables change if there is a conflict) - @param [in] mDebug= set to 1 to show debug messages in the log - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createdocument( - tree=/User Folders/sasdemo - ,name=myNote - ,desc=Created by &sysmacroname - ,textrole= - ,frefin=mm_in - ,frefout=mm_out - ,mDebug=1 - ); - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -%mp_abort(iftrue= (%mf_verifymacvars(tree name)=0) - ,mac=&sysmacroname - ,msg=%str(Empty inputs: tree name) -) - -/** - * check tree exists - */ - -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree","Folder",type,uri); - call symputx('type',type,'l'); - call symputx('treeuri',uri,'l'); -run; - -%mp_abort( - iftrue= (&type ne Tree) - ,mac=mm_createdocument.sas - ,msg=Tree &tree does not exist! -) - -/** - * Check object does not exist already - */ -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree/&name","Note",type,uri); - call symputx('type',type,'l'); - call symputx('docuri',uri,'l'); - putlog (_all_)(=); -run; - -%if &type = Document %then %do; - %put Document &name already exists in &tree!; - %return; -%end; - -/** - * Now we can create the document - */ -filename &frefin temp; - -/* write header XML */ -data _null_; - file &frefin; - name=quote("&name"); - desc=quote("&desc"); - textrole=quote("&textrole"); - treeuri=quote("&treeuri"); - - put "$METAREPOSITORY"/ - '"/ - " "/ - ' ' / - ''/ - /*URI="Document for public note" */ - ""/ - "SAS"/ - "268435456"; -run; - -filename &frefout temp; - -proc metadata in= &frefin out=&frefout verbose; -run; - -%if &mdebug=1 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile &frefout lrecl=1048576; - input; - put _infile_; - run; -%end; - -%mend mm_createdocument; diff --git a/003_macros/mm_createfolder.sas b/003_macros/mm_createfolder.sas deleted file mode 100644 index 161704f..0000000 --- a/003_macros/mm_createfolder.sas +++ /dev/null @@ -1,161 +0,0 @@ -/*** HELP START ***//** - @file - @brief Recursively create a metadata folder - @details This macro was inspired by Paul Homes who wrote an early - version (mkdirmd.sas) in 2010. The original is described here: - https://platformadmin.com/blogs/paul/2010/07/mkdirmd/ - - The macro will NOT create a new ROOT folder - not - because it can't, but more because that is generally not something - your administrator would like you to do! - - The macro is idempotent - if you run it twice, it will only create a folder - once. - - Usage: - - %mm_createfolder(path=/some/meta/folder) - - @param [in] path= Name of the folder to create. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createfolder(path=,mDebug=0); -%put &sysmacroname: execution started for &path; -%local dbg errorcheck; -%if &mDebug=0 %then %let dbg=*; - -%local parentFolderObjId child errorcheck paths; -%let paths=0; -%let errorcheck=1; - -%if &syscc ge 4 %then %do; - %put SYSCC=&syscc - this macro requires a clean session; - %return; -%end; - -data _null_; - length objId parentId objType parent child $200 - folderPath $1000; - call missing (of _all_); - folderPath = "%trim(&path)"; - - * remove any trailing slash ; - if ( substr(folderPath,length(folderPath),1) = '/' ) then - folderPath=substr(folderPath,1,length(folderPath)-1); - - * name must not be blank; - if ( folderPath = '' ) then do; - put 'ERR' +(-1) "OR: &sysmacroname PATH parameter value must be non-blank"; - end; - - * must have a starting slash ; - if ( substr(folderPath,1,1) ne '/' ) then do; - put 'ERR' +(-1) "OR: &sysmacroname PATH param value must have starting slash"; - stop; - end; - - * check if folder already exists ; - rc=metadata_pathobj('',cats(folderPath,"(Folder)"),"",objType,objId); - if rc ge 1 then do; - put "NOTE: Folder " folderPath " already exists!"; - stop; - end; - - * do not create a root (one level) folder ; - if countc(folderPath,'/')=1 then do; - put 'ERR' +(-1) "OR: &sysmacroname will not create a new ROOT folder"; - stop; - end; - - * check that root folder exists ; - root=cats('/',scan(folderpath,1,'/'),"(Folder)"); - if metadata_pathobj('',root,"",objType,parentId)<1 then do; - put 'ERR' +(-1) "OR: " root " does not exist!"; - stop; - end; - - * check that parent folder exists ; - child=scan(folderPath,-1,'/'); - parent=substr(folderpath,1,length(folderpath)-length(child)-1); - rc=metadata_pathobj('',cats(parent,"(Folder)"),"",objType,parentId); - if rc<1 then do; - putlog 'The following folders will be created:'; - /* folder does not exist - so start from top and work down */ - length newpath $1000; - paths=0; - do x=2 to countw(folderpath,'/'); - newpath=''; - do i=1 to x; - newpath=cats(newpath,'/',scan(folderpath,i,'/')); - end; - rc=metadata_pathobj('',cats(newpath,"(Folder)"),"",objType,parentId); - if rc<1 then do; - paths+1; - call symputx(cats('path',paths),newpath); - putlog newpath; - end; - call symputx('paths',paths); - end; - end; - else putlog "parent " parent " exists"; - - call symputx('parentFolderObjId',parentId,'l'); - call symputx('child',child,'l'); - call symputx('errorcheck',0,'l'); - - &dbg put (_all_)(=); -run; - -%if &errorcheck=1 or &syscc ge 4 %then %return; - -%if &paths>0 %then %do x=1 %to &paths; - %put executing recursive call for &&path&x; - %mm_createfolder(path=&&path&x) -%end; -%else %do; - filename __newdir temp; - options noquotelenmax; - %local inmeta; - %put creating: &path; - %let inmeta=$METAREPOSITORY - - SAS268435456 - ; - - proc metadata in="&inmeta" out=__newdir verbose; - run ; - - /* check it was successful */ - data _null_; - length objId parentId objType parent child $200 ; - call missing (of _all_); - rc=metadata_pathobj('',cats("&path","(Folder)"),"",objType,objId); - if rc ge 1 then do; - putlog "SUCCCESS! &path created."; - end; - else do; - putlog 'ERR' +(-1) "OR: unsuccessful attempt to create &path"; - call symputx('syscc',8); - end; - run; - - /* write the response to the log for debugging */ - %if &mDebug ne 0 %then %do; - data _null_; - infile __newdir lrecl=32767; - input; - put _infile_; - run; - %end; - filename __newdir clear; -%end; - -%put &sysmacroname: execution finished for &path; -%mend mm_createfolder; diff --git a/003_macros/mm_createlibrary.sas b/003_macros/mm_createlibrary.sas deleted file mode 100644 index 84a9232..0000000 --- a/003_macros/mm_createlibrary.sas +++ /dev/null @@ -1,326 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a SAS Library - @details Currently only supports BASE engine - - This macro is idempotent - if you run it twice (for the same libref or - libname), it will only create one library. There is a dependency on other - macros in this library - they should be installed as a suite (see README). - - Usage: - - %mm_createlibrary( - libname=My New Library - ,libref=mynewlib - ,libdesc=Super & - ,engine=BASE - ,tree=/User Folders/sasdemo - ,servercontext=SASApp - ,directory=/tmp/tests - ,mDebug=1) - -

SAS Macros

- @li mf_verifymacvars.sas - @li mm_createfolder.sas - - - @param [in] libname= (My New Library) Library name (as displayed to user, - 256 chars). Duplicates are not created (case sensitive). - @param [in] libref= (mynewlib) Library libref (8 chars). Duplicate - librefs are not created, HOWEVER- the check is not case sensitive - if - *libref* exists, *LIBREF* will still be created. - Librefs created will always be uppercased. - @param [in] engine= Library engine (currently only BASE supported) - @param [in] tree= The metadata folder uri, or the metadata path, in which to - create the library. - @param [in] servercontext= (SASApp) The SAS server against which - the library is registered. - @param [in] IsPreassigned= set to 1 if the library should be pre-assigned. - - @param [in] libdesc= Library description (optional) - @param [in] directory= (/tmp/somelib) Required for the BASE engine. - The metadata directory objects are searched to find an existing - one with a matching physical path. - If more than one uri found with that path, then the first one will be used. - If no URI is found, a new directory object will be created. The physical - path will also be created, if it doesn't exist. - - - @param [in] mDebug= set to 1 to show debug messages in the log - @param [in] frefin= fileref to use (enables change if there is a conflict). - The filerefs are left open, to enable inspection after running the - macro (or importing into an xmlmap if needed). - @param [out] frefout= fileref to use (enables change if there is a conflict) - - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createlibrary( - libname=My New Library - ,libref=mynewlib - ,libdesc=Created automatically using the mm_createlibrary macro - ,engine=BASE - ,tree=/User Folders/sasdemo - ,servercontext=SASApp - ,directory=/tmp/somelib - ,IsPreassigned=0 - ,mDebug=0 - ,frefin=mm_in - ,frefout=mm_out -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -%let libref=%upcase(&libref); - -/** - * Check Library does not exist already with this libname - */ -data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:SASLibrary?@Name='&libname'",type,uri); - call symputx('checktype',type,'l'); - call symputx('liburi',uri,'l'); - putlog (_all_)(=); -run; -%if &checktype = SASLibrary %then %do; - %put %str(WARN)ING: Library (&liburi) already exists with libname (&libname); - %return; -%end; - -/** - * Check Library does not exist already with this libref - */ -data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:SASLibrary?@Libref='&libref'",type,uri); - call symputx('checktype',type,'l'); - call symputx('liburi',uri,'l'); - putlog (_all_)(=); -run; -%if &checktype = SASLibrary %then %do; - %put %str(WARN)ING: Library (&liburi) already exists with libref (&libref) ; - %return; -%end; - - -/** - * Attempt to create tree - */ -%mm_createfolder(path=&tree) - -/** - * check tree exists - */ -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree","Folder",type,uri); - call symputx('foldertype',type,'l'); - call symputx('treeuri',uri,'l'); -run; -%if &foldertype ne Tree %then %do; - %put %str(WARN)ING: Tree &tree does not exist!; - %return; -%end; - -/** - * Create filerefs for proc metadata call - */ -filename &frefin temp; -filename &frefout temp; - -%mp_abort(iftrue= ( - &engine=BASE & %mf_verifymacvars(libname libref engine servercontext tree)=0 - ) - ,mac=&sysmacroname - ,msg=%str(Empty inputs: libname libref engine servercontext tree) -) - -%if &engine=BASE %then %do; - /** - * Check that the ServerContext exists - */ - data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:ServerContext?@Name='&ServerContext'",type,uri); - call symputx('checktype',type,'l'); - call symputx('serveruri',uri,'l'); - putlog (_all_)(=); - run; - %if &checktype ne ServerContext %then %do; - %put %str(ERR)OR: ServerContext (&ServerContext) does not exist!; - %return; - %end; - - /** - * Get prototype info - */ - data _null_; - length type uri str $256; - str="omsobj:Prototype?@Name='Library.SAS.Prototype.Name.xmlKey.txt'"; - rc=metadata_resolve(str,type,uri); - call symputx('checktype',type,'l'); - call symputx('prototypeuri',uri,'l'); - putlog (_all_)(=); - run; - %if &checktype ne Prototype %then %do; - %put %str(ERR)OR: Prototype Library.SAS.Prototype.Name.xmlKey.txt not found; - %return; - %end; - - /** - * Check that Physical location exists - */ - %if %sysfunc(fileexist(&directory))=0 %then %do; - %put %str(ERR)OR: Physical directory (&directory) does not appear to exist!; - %return; - %end; - - /** - * Check that Directory Object exists in metadata - */ - data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:Directory?@DirectoryRole='LibraryPath'" - !!" and @DirectoryName='&directory'",type,uri); - call symputx('checktype',type,'l'); - call symputx('directoryuri',uri,'l'); - putlog (_all_)(=); - run; - %if &checktype ne Directory %then %do; - %put NOTE: Directory object does not exist for (&directory) location; - %put NOTE: It will now be created; - - data _null_; - file &frefin; - directory=quote(symget('directory')); - put "$METAREPOSITORY "/ - ''/ - "SAS"/ - "268435456"; - run; - - proc metadata in= &frefin out=&frefout %if &mdebug=1 %then verbose;; - run; - %if &mdebug=1 %then %do; - data _null_; - infile &frefout lrecl=1048576; - input; put _infile_; - run; - %end; - %put NOTE: Checking to ensure directory (&directory) object was created; - data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:Directory?@DirectoryRole='LibraryPath'" - !!" and @DirectoryName='&directory'",type,uri); - call symputx('checktype2',type,'l'); - call symputx('directoryuri',uri,'l'); - %if &mdebug=1 %then putlog (_all_)(=);; - run; - %if &checktype2 ne Directory %then %do; - %put %str(ERR)OR: Directory (&directory) object was NOT created!; - %return; - %end; - %else %put NOTE: Directory (&directoryuri) successfully created!; - %end; - - /** - * check SAS version - */ - %if %sysevalf(&sysver lt 9.3) %then %do; - %put %str(WARN)ING: Version 9.3 or later required; - %return; - %end; - - /** - * Prepare the XML and create the library - */ - data _null_; - file &frefin; - treeuri=quote(symget('treeuri')); - serveruri=quote(symget('serveruri')); - directoryuri=quote(symget('directoryuri')); - libname=quote(symget('libname')); - libref=quote(symget('libref')); - IsPreassigned=quote(symget('IsPreassigned')); - prototypeuri=quote(symget('prototypeuri')); - - /* escape description so it can be stored as XML */ - libdesc=tranwrd(symget('libdesc'),'&','&'); - libdesc=tranwrd(libdesc,'<','<'); - libdesc=tranwrd(libdesc,'>','>'); - libdesc=tranwrd(libdesc,"'",'''); - libdesc=tranwrd(libdesc,'"','"'); - libdesc=tranwrd(libdesc,'0A'x,' '); - libdesc=tranwrd(libdesc,'0D'x,' '); - libdesc=tranwrd(libdesc,'$','$'); - libdesc=quote(trim(libdesc)); - - put "$METAREPOSITORY "/ - ''/ - ' '/ - ' '/ - ' '/ - ' '/ - ' '/ - " "/ - ' '/ - ' '/ - ' '/ - ' '/ - ' '/ - ' '/ - 'SAS'/ - '268435456'; - run; - - - proc metadata in= &frefin out=&frefout %if &mdebug=1 %then verbose ;; - run; - - %if &mdebug=1 %then %do; - data _null_; - infile &frefout lrecl=1048576; - input;put _infile_; - run; - %end; - %put NOTE: Checking to ensure library (&libname) was created; - data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree/&libname","Library",type,uri); - call symputx('libtype',type,'l'); - call symputx('liburi',uri,'l'); - %if &mdebug=1 %then putlog (_all_)(=);; - run; - %if &libtype ne SASLibrary %then %do; - %put %str(ERR)OR: Could not find (&libname) at (&tree)!!; - %return; - %end; - %else %put NOTE: Library (&libname) successfully created in (&tree)!; -%end; -%else %do; - %put %str(ERR)OR: Other library engine types are not yet supported!!; -%end; - - -/** - * Wrap up - */ -%if &mdebug ne 1 %then %do; - filename &frefin clear; - filename &frefout clear; -%end; - -%mend mm_createlibrary; diff --git a/003_macros/mm_createwebservice.sas b/003_macros/mm_createwebservice.sas deleted file mode 100644 index 4fd2ac0..0000000 --- a/003_macros/mm_createwebservice.sas +++ /dev/null @@ -1,711 +0,0 @@ -/*** HELP START ***//** - @file mm_createwebservice.sas - @brief Create a Web Ready Stored Process - @details This macro creates a Type 2 Stored Process with the mm_webout macro - (and dependencies) included as pre-code. - -Usage: - - %* compile macros ; - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* parmcards lets us write to a text file from open code ; - filename ft15f001 temp; - parmcards4; - %webout(FETCH) - %* do some sas, any inputs are now already WORK tables; - data example1 example2; - set sashelp.class; - run; - %* send data back; - %webout(OPEN) - %webout(ARR,example1) * Array format, fast, suitable for large tables ; - %webout(OBJ,example2) * Object format, easier to work with ; - %webout(CLOSE) - ;;;; - %mm_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001) - - For more examples of using these web services with the SASjs Adapter, see: - https://github.com/sasjs/adapter#readme - - @param [in] path= () The full path (in SAS Metadata) where the service - will be created - @param [in] name= Stored Process name. Avoid spaces - testing has shown that - the check to avoid creating multiple STPs in the same folder with the same - name does not work when the name contains spaces. - @param [in] desc= The description of the service (optional) - @param [in] precode= () Space separated list of filerefs, pointing to the - code that needs to be attached to the beginning of the service (optional) - @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code - to be added - @param [in] server= (SASApp) The server which will run the STP. Server - name or uri is fine. - @param [in] mDebug= (0) set to 1 to show debug messages in the log - @param [in] replace= (YES) select NO to avoid replacing an existing service - in that location - @param [in] adapter= (sasjs) the macro uses the sasjs adapter by default. - To use another adapter, add a (different) fileref here. - -

SAS Macros

- @li mm_createstp.sas - @li mf_getuser.sas - @li mm_createfolder.sas - @li mm_deletestp.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_createwebservice(path= - ,name=initService - ,precode= - ,code=ft15f001 - ,desc=This stp was created automagically by the mm_createwebservice macro - ,mDebug=0 - ,server=SASApp - ,replace=YES - ,adapter=sasjs -)/*/STORE SOURCE*/; - -%if &syscc ge 4 %then %do; - %put &=syscc - &sysmacroname will not execute in this state; - %return; -%end; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_createwebservice.sas; -%&mD.put _local_; - -* remove any trailing slash ; -%if "%substr(&path,%length(&path),1)" = "/" %then - %let path=%substr(&path,1,%length(&path)-1); - -/** - * Add webout macro - * These put statements are auto generated - to change the macro, change the - * source (mm_webout) and run `build.py` - */ -filename sasjs temp; -data _null_; - file sasjs lrecl=3000 ; - put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */"; -/* WEBOUT BEGIN */ - put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y '; - put ' ,engine=DATASTEP '; - put ' ,missing=NULL '; - put ' ,showmeta=N '; - put ' ,maxobs=MAX '; - put ')/*/STORE SOURCE*/; '; - put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval '; - put ' tmpds1 tmpds2 tmpds3 tmpds4; '; - put '%let numcols=0; '; - put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); '; - put ' '; - put '%if &action=OPEN %then %do; '; - put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; - put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; - put ' run; '; - put '%end; '; - put '%else %if (&action=ARR or &action=OBJ) %then %do; '; - put ' /* force variable names to always be uppercase in the JSON */ '; - put ' options validvarname=upcase; '; - put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; - put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; - put ' filename _sjs1 temp lrecl=200 ; '; - put ' data _null_; file _sjs1 encoding=''utf-8''; '; - put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; - put ' run; '; - put ' /* now write to _webout 1 char at a time */ '; - put ' data _null_; '; - put ' infile _sjs1 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs1 clear; '; - put ' '; - put ' /* grab col defs */ '; - put ' proc contents noprint data=&ds '; - put ' out=_data_(keep=name type length format formatl formatd varnum label); '; - put ' run; '; - put ' %let colinfo=%scan(&syslast,2,.); '; - put ' proc sort data=&colinfo; '; - put ' by varnum; '; - put ' run; '; - put ' /* move meta to mac vars */ '; - put ' data &colinfo; '; - put ' if _n_=1 then call symputx(''numcols'',nobs,''l''); '; - put ' set &colinfo end=last nobs=nobs; '; - put ' name=upcase(name); '; - put ' /* fix formats */ '; - put ' if type=2 or type=6 then do; '; - put ' typelong=''char''; '; - put ' length fmt $49.; '; - put ' if format='''' then fmt=cats(''$'',length,''.''); '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else fmt=cats(format,formatl,''.''); '; - put ' end; '; - put ' else do; '; - put ' typelong=''num''; '; - put ' if format='''' then fmt=''best.''; '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else if formatd=0 then fmt=cats(format,formatl,''.''); '; - put ' else fmt=cats(format,formatl,''.'',formatd); '; - put ' end; '; - put ' /* 32 char unique name */ '; - put ' newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); '; - put ' '; - put ' call symputx(cats(''name'',_n_),name,''l''); '; - put ' call symputx(cats(''newname'',_n_),newname,''l''); '; - put ' call symputx(cats(''length'',_n_),length,''l''); '; - put ' call symputx(cats(''fmt'',_n_),fmt,''l''); '; - put ' call symputx(cats(''type'',_n_),type,''l''); '; - put ' call symputx(cats(''typelong'',_n_),typelong,''l''); '; - put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; - put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; - put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; - put ' run; '; - put ' '; - put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql; '; - put ' select count(*) into: lastobs from &ds; '; - put ' %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); '; - put ' '; - put ' %if &engine=PROCJSON %then %do; '; - put ' %if &missing=STRING %then %do; '; - put ' %put &sysmacroname: Special Missings not supported in proc json.; '; - put ' %put &sysmacroname: Switching to DATASTEP engine; '; - put ' %goto datastep; '; - put ' %end; '; - put ' data &tempds; '; - put ' set &ds; '; - put ' &stmt_obs; '; - put ' %if &fmt=N %then format _numeric_ best32.;; '; - put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; - put ' proc json out=_sjs2 pretty '; - put ' %if &action=ARR %then nokeys ; '; - put ' ;export &tempds / nosastags fmtnumeric; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs2 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs2 clear; '; - put ' %end; '; - put ' %else %if &engine=DATASTEP %then %do; '; - put ' %datastep: '; - put ' %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 '; - put ' %then %do; '; - put ' %put &sysmacroname: &ds NOT FOUND!!!; '; - put ' %return; '; - put ' %end; '; - put ' '; - put ' %if &fmt=Y %then %do; '; - put ' /** '; - put ' * Extract format definitions '; - put ' * First, by getting library locations from dictionary.formats '; - put ' * Then, by exporting the width using proc format '; - put ' * Cannot use maxw from sashelp.vformat as not always populated '; - put ' * Cannot use fmtinfo() as not supported in all flavours '; - put ' */ '; - put ' %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql noprint; '; - put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as FMTCAT, '; - put ' FMTNAME '; - put ' from dictionary.formats '; - put ' where fmttype=''F'' and libname is not null '; - put ' and fmtname in (select format from &colinfo where format is not null) '; - put ' order by 1; '; - put ' create table &tmpds2( '; - put ' FMTNAME char(32), '; - put ' LENGTH num '; - put ' ); '; - put ' %local catlist cat fmtlist i; '; - put ' select distinct fmtcat into: catlist separated by '' '' from &tmpds1; '; - put ' %do i=1 %to %sysfunc(countw(&catlist,%str( ))); '; - put ' %let cat=%scan(&catlist,&i,%str( )); '; - put ' proc sql; '; - put ' select distinct fmtname into: fmtlist separated by '' '' '; - put ' from &tmpds1 where fmtcat="&cat"; '; - put ' proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); '; - put ' select &fmtlist; '; - put ' run; '; - put ' proc sql; '; - put ' insert into &tmpds2 select distinct fmtname,length from &tmpds3; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' create table &tmpds4 as '; - put ' select a.*, b.length as MAXW '; - put ' from &colinfo a '; - put ' left join &tmpds2 b '; - put ' on cats(a.format)=cats(upcase(b.fmtname)) '; - put ' order by a.varnum; '; - put ' data _null_; '; - put ' set &tmpds4; '; - put ' if not missing(maxw); '; - put ' call symputx( '; - put ' cats(''fmtlen'',_n_), '; - put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; - put ' ,''l'' '; - put ' ); '; - put ' run; '; - put ' '; - put ' /* configure varlenchk - as we are explicitly shortening the variables */ '; - put ' %let optval=%sysfunc(getoption(varlenchk)); '; - put ' options varlenchk=NOWARN; '; - put ' data _data_(compress=char); '; - put ' /* shorten the new vars */ '; - put ' length '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i $&&fmtlen&i '; - put ' %end; '; - put ' ; '; - put ' /* rename on entry */ '; - put ' set &ds(rename=( '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i=&&newname&i '; - put ' %end; '; - put ' )); '; - put ' &stmt_obs; '; - put ' '; - put ' drop '; - put ' %do i=1 %to &numcols; '; - put ' &&newname&i '; - put ' %end; '; - put ' ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=num %then %do; '; - put ' &&name&i=cats(put(&&newname&i,&&fmt&i)); '; - put ' %end; '; - put ' %else %do; '; - put ' &&name&i=put(&&newname&i,&&fmt&i); '; - put ' %end; '; - put ' %end; '; - put ' if _error_ then do; '; - put ' call symputx(''syscc'',1012); '; - put ' stop; '; - put ' end; '; - put ' run; '; - put ' %let fmtds=&syslast; '; - put ' options varlenchk=&optval; '; - put ' %end; '; - put ' '; - put ' proc format; /* credit yabwon for special null removal */ '; - put ' value bart (default=40) '; - put ' %if &missing=NULL %then %do; '; - put ' ._ - .z = null '; - put ' %end; '; - put ' %else %do; '; - put ' ._ = [quote()] '; - put ' . = null '; - put ' .a - .z = [quote()] '; - put ' %end; '; - put ' other = [best.]; '; - put ' '; - put ' data &tempds; '; - put ' attrib _all_ label=''''; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' length &&name&i $&&fmtlen&i...; '; - put ' format &&name&i $&&fmtlen&i...; '; - put ' %end; '; - put ' %end; '; - put ' %if &fmt=Y %then %do; '; - put ' set &fmtds; '; - put ' %end; '; - put ' %else %do; '; - put ' set &ds; '; - put ' %end; '; - put ' &stmt_obs; '; - put ' format _numeric_ bart.; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' &&name&i=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; - put ' %end; '; - put ' %end; '; - put ' run; '; - put ' '; - put ' filename _sjs3 temp lrecl=131068 ; '; - put ' data _null_; '; - put ' file _sjs3 encoding=''utf-8''; '; - put ' if _n_=1 then put "["; '; - put ' set &tempds; '; - put ' if _n_>1 then put "," @; put '; - put ' %if &action=ARR %then "[" ; %else "{" ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &i>1 %then "," ; '; - put ' %if &action=OBJ %then """&&name&i"":" ; '; - put ' "&&name&i"n /* name literal for reserved variable names */ '; - put ' %end; '; - put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file _sjs3 mod encoding=''utf-8''; '; - put ' put '']''; '; - put ' run; '; - put ' data _null_; '; - put ' infile _sjs3 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs3 clear; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' drop table &colinfo, &tempds; '; - put ' '; - put ' %if %substr(&showmeta,1,1)=Y %then %do; '; - put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; '; - put ' file _sjs4; '; - put ' length label $350; '; - put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; - put ' do i=1 to &numcols; '; - put ' name=quote(trim(symget(cats(''name'',i)))); '; - put ' format=quote(trim(symget(cats(''fmt'',i)))); '; - put ' label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); '; - put ' length=quote(trim(symget(cats(''length'',i)))); '; - put ' type=quote(trim(symget(cats(''typelong'',i)))); '; - put ' if i>1 then put "," @@; '; - put ' put name '':{"format":'' format '',"label":'' label '; - put ' '',"length":'' length '',"type":'' type ''}''; '; - put ' end; '; - put ' put ''}}''; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs4 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs4 clear; '; - put ' %end; '; - put '%end; '; - put ' '; - put '%else %if &action=CLOSE %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod ; '; - put ' put "}"; '; - put ' run; '; - put '%end; '; - put '%mend mp_jsonout; '; - put ' '; - put '%macro mf_getuser( '; - put ')/*/STORE SOURCE*/; '; - put ' %local user; '; - put ' '; - put ' %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; '; - put ' %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; '; - put ' %let user=&SYS_COMPUTE_SESSION_OWNER; '; - put ' %end; '; - put ' %else %if %symexist(_metaperson) %then %do; '; - put ' %if %length(&_metaperson)=0 %then %let user=&sysuserid; '; - put ' /* sometimes SAS will add @domain extension - remove for consistency */ '; - put ' /* but be sure to quote in case of usernames with commas */ '; - put ' %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); '; - put ' %end; '; - put ' %else %let user=&sysuserid; '; - put ' '; - put ' %quote(&user) '; - put ' '; - put '%mend mf_getuser; '; - put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL '; - put ' ,showmeta=N,maxobs=MAX,workobs=0 '; - put '); '; - put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug '; - put ' sasjs_tables; '; - put '%local i tempds jsonengine; '; - put ' '; - put '/* see https://github.com/sasjs/core/issues/41 */ '; - put '%if "%upcase(&SYSENCODING)" ne "UTF-8" %then %let jsonengine=PROCJSON; '; - put '%else %let jsonengine=DATASTEP; '; - put ' '; - put ' '; - put '%if &action=FETCH %then %do; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' options mprint notes mprintnest; '; - put ' %end; '; - put ' %let _webin_file_count=%eval(&_webin_file_count+0); '; - put ' /* now read in the data */ '; - put ' %do i=1 %to &_webin_file_count; '; - put ' %if &_webin_file_count=1 %then %do; '; - put ' %let _webin_fileref1=&_webin_fileref; '; - put ' %let _webin_name1=&_webin_name; '; - put ' %end; '; - put ' data _null_; '; - put ' infile &&_webin_fileref&i termstr=crlf; '; - put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; - put ' putlog "&&_webin_name&i input statement: " _infile_; '; - put ' stop; '; - put ' data &&_webin_name&i; '; - put ' infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8''; '; - put ' input &input_statement; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' if _n_<20 then putlog _infile_; '; - put ' %end; '; - put ' run; '; - put ' %let sasjs_tables=&sasjs_tables &&_webin_name&i; '; - put ' %end; '; - put '%end; '; - put ' '; - put '%else %if &action=OPEN %then %do; '; - put ' /* fix encoding */ '; - put ' OPTIONS NOBOMFILE; '; - put ' '; - put ' /** '; - put ' * check xengine type to avoid the below err message: '; - put ' * > Function is only valid for filerefs using the CACHE access method. '; - put ' */ '; - put ' data _null_; '; - put ' set sashelp.vextfl(where=(fileref="_WEBOUT")); '; - put ' if xengine=''STREAM'' then do; '; - put ' rc=stpsrv_header(''Content-type'',"text/html; encoding=utf-8"); '; - put ' end; '; - put ' run; '; - put ' '; - put ' /* setup json */ '; - put ' data _null_;file &fref encoding=''utf-8''; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' put ''>>weboutBEGIN<<''; '; - put ' %end; '; - put ' put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; '; - put ' put '',"SYSTIME" : "'' "&SYSTIME" ''"''; '; - put ' run; '; - put ' '; - put '%end; '; - put ' '; - put '%else %if &action=ARR or &action=OBJ %then %do; '; - put ' %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref '; - put ' ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs '; - put ' ) '; - put '%end; '; - put '%else %if &action=CLOSE %then %do; '; - put ' /* To avoid issues with _webout on EBI we use a temporary file */ '; - put ' filename _sjsref temp lrecl=131068; '; - put ' %if %str(&workobs) > 0 %then %do; '; - put ' /* if debug mode, send back first XX records of each work table also */ '; - put ' data;run;%let tempds=%scan(&syslast,2,.); '; - put ' ods output Members=&tempds; '; - put ' proc datasets library=WORK memtype=data; '; - put ' %local wtcnt;%let wtcnt=0; '; - put ' data _null_; '; - put ' set &tempds; '; - put ' if not (upcase(name) =:"DATA"); /* ignore temp datasets */ '; - put ' i+1; '; - put ' call symputx(cats(''wt'',i),name,''l''); '; - put ' call symputx(''wtcnt'',i,''l''); '; - put ' data _null_; file _sjsref mod encoding=''utf-8''; '; - put ' put ",""WORK"":{"; '; - put ' %do i=1 %to &wtcnt; '; - put ' %let wt=&&wt&i; '; - put ' data _null_; file _sjsref mod encoding=''utf-8''; '; - put ' dsid=open("WORK.&wt",''is''); '; - put ' nlobs=attrn(dsid,''NLOBS''); '; - put ' nvars=attrn(dsid,''NVARS''); '; - put ' rc=close(dsid); '; - put ' if &i>1 then put '',''@; '; - put ' put " ""&wt"" : {"; '; - put ' put ''"nlobs":'' nlobs; '; - put ' put '',"nvars":'' nvars; '; - put ' %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y '; - put ' ,maxobs=&workobs '; - put ' ) '; - put ' data _null_; file _sjsref mod encoding=''utf-8''; '; - put ' put "}"; '; - put ' %end; '; - put ' data _null_; file _sjsref mod encoding=''utf-8''; '; - put ' put "}"; '; - put ' run; '; - put ' %end; '; - put ' /* close off json */ '; - put ' data _null_;file _sjsref mod encoding=''utf-8''; '; - put ' length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; '; - put ' put ",""_DEBUG"" : ""&_debug"" "; '; - put ' _METAUSER=quote(trim(symget(''_METAUSER''))); '; - put ' put ",""_METAUSER"": " _METAUSER; '; - put ' _METAPERSON=quote(trim(symget(''_METAPERSON''))); '; - put ' put '',"_METAPERSON": '' _METAPERSON; '; - put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; - put ' put '',"_PROGRAM" : '' _PROGRAM ; '; - put ' autoexec=quote(urlencode(trim(getoption(''autoexec'')))); '; - put ' put '',"AUTOEXEC" : '' autoexec; '; - put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; - put ' put ",""SYSCC"" : ""&syscc"" "; '; - put ' put ",""SYSENCODING"" : ""&sysencoding"" "; '; - put ' syserrortext=cats(symget(''syserrortext'')); '; - put ' if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syserrortext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syserrortext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syserrortext=cats(''"'',syserrortext,''"''); '; - put ' put '',"SYSERRORTEXT" : '' syserrortext; '; - put ' put ",""SYSHOSTNAME"" : ""&syshostname"" "; '; - put ' put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; '; - put ' put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; '; - put ' SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); '; - put ' put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; '; - put ' put ",""SYSJOBID"" : ""&sysjobid"" "; '; - put ' put ",""SYSSCPL"" : ""&sysscpl"" "; '; - put ' put ",""SYSSITE"" : ""&syssite"" "; '; - put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; - put ' sysvlong=quote(trim(symget(''sysvlong''))); '; - put ' put '',"SYSVLONG" : '' sysvlong; '; - put ' syswarningtext=cats(symget(''syswarningtext'')); '; - put ' if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syswarningtext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syswarningtext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syswarningtext=cats(''"'',syswarningtext,''"''); '; - put ' put '',"SYSWARNINGTEXT" : '' syswarningtext; '; - put ' put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; '; - put ' length memsize $32; '; - put ' memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; '; - put ' memsize=quote(cats(memsize)); '; - put ' put '',"MEMSIZE" : '' memsize; '; - put ' put "}" @; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' put ''>>weboutEND<<''; '; - put ' %end; '; - put ' run; '; - put ' /* now write to _webout 1 char at a time */ '; - put ' data _null_; '; - put ' infile _sjsref lrecl=1 recfm=n; '; - put ' file &fref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjsref clear; '; - put ' '; - put '%end; '; - put ' '; - put '%mend mm_webout; '; -/* WEBOUT END */ - put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO'; - put ' ,maxobs=MAX'; - put ');'; - put ' %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing'; - put ' ,showmeta=&showmeta,maxobs=&maxobs'; - put ' )'; - put '%mend;'; -run; - -/* add precode and code */ -%local work tmpfile; -%let work=%sysfunc(pathname(work)); -%let tmpfile=__mm_createwebservice.temp; -%local x fref freflist mod; -%let freflist= &adapter &precode &code ; -%do x=1 %to %sysfunc(countw(&freflist)); - %if &x>1 %then %let mod=mod; - - %let fref=%scan(&freflist,&x); - %&mD.put &sysmacroname: adding &fref; - data _null_; - file "&work/&tmpfile" lrecl=3000 &mod; - infile &fref; - input; - put _infile_; - run; -%end; - -/* create the metadata folder if not already there */ -%mm_createfolder(path=&path) -%if &syscc ge 4 %then %return; - -%if %upcase(&replace)=YES %then %do; - %mm_deletestp(target=&path/&name) -%end; - -/* create the web service */ -%mm_createstp(stpname=&name - ,filename=&tmpfile - ,directory=&work - ,tree=&path - ,stpdesc=&desc - ,mDebug=&mdebug - ,server=&server - ,stptype=2) - -/* find the web app url */ -%local url; -%let url=localhost/SASStoredProcess; -data _null_; - length url $128; - rc=METADATA_GETURI("Stored Process Web App",url); - if rc=0 then call symputx('url',url,'l'); -run; - -%put &sysmacroname: STP &name successfully created in &path; -%put Check it out here:; -%put ;%put ;%put ; -%put &url?_PROGRAM=&path/&name; -%put ;%put ;%put ; - -%mend mm_createwebservice; diff --git a/003_macros/mm_deletedocument.sas b/003_macros/mm_deletedocument.sas deleted file mode 100644 index 7a3b72c..0000000 --- a/003_macros/mm_deletedocument.sas +++ /dev/null @@ -1,71 +0,0 @@ -/*** HELP START ***//** - @file mm_deletedocument.sas - @brief Deletes a Document using path as reference - @details - - Usage: - - %mm_createdocument(tree=/User Folders/&sysuserid,name=MyNote) - %mm_deletedocument(target=/User Folders/&sysuserid/MyNote) - -

SAS Macros

- - @param [in] target= full path to the document being deleted - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_deletedocument( - target= -)/*/STORE SOURCE*/; - -/** - * Check document exist - */ -%local type; -data _null_; - length type uri $256; - rc=metadata_pathobj("","&target",'Note',type,uri); - call symputx('type',type,'l'); - call symputx('stpuri',uri,'l'); -run; -%if &type ne Document %then %do; - %put %str(WARN)ING: No Document found at ⌖ - %return; -%end; - -filename __in temp lrecl=10000; -filename __out temp lrecl=10000; -data _null_ ; - file __in ; - put ""; - put "SAS268436480"; - put ""; -run ; -proc metadata in=__in out=__out verbose;run; - -/* list the result */ -data _null_;infile __out; input; list; run; - -filename __in clear; -filename __out clear; - -/** - * Check deletion - */ -%local isgone; -data _null_; - length type uri $256; - call missing (of _all_); - rc=metadata_pathobj("","&target",'Note',type,uri); - call symputx('isgone',type,'l'); -run; -%if &isgone = Document %then %do; - %put %str(ERR)OR: Document not deleted from ⌖ - %let syscc=4; - %return; -%end; - -%mend mm_deletedocument; diff --git a/003_macros/mm_deletelibrary.sas b/003_macros/mm_deletelibrary.sas deleted file mode 100644 index 892fc9e..0000000 --- a/003_macros/mm_deletelibrary.sas +++ /dev/null @@ -1,93 +0,0 @@ -/*** HELP START ***//** - @file - @brief Deletes a library by Name - - @details Used to delete a library. - Usage: - - %* create a library in the home directory ; - %mm_createlibrary( - libname=My Temp Library, - libref=XXTEMPXX, - tree=/User Folders/&sysuserid, - directory=%sysfunc(pathname(work)) - ) - - %* delete the library ; - %mm_deletelibrary(name=My Temp Library) - - After running the above, the following will be shown in the log: - - ![](https://i.imgur.com/Y4Tog24.png) - - @param [in] name= () the name (not libref) of the library to be deleted - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mp_abort.sas - - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_deletelibrary( - name= -)/*/STORE SOURCE*/; - - -/** - * Check if library exists and get uri - */ -data _null_; - length type uri $256; - rc=metadata_resolve("omsobj:SASLibrary?@Name='&name'",type,uri); - call symputx('checktype',type,'l'); - call symputx('liburi',uri,'l'); - putlog (_all_)(=); -run; -%if &checktype ne SASLibrary %then %do; - %put &sysmacroname: Library (&name) was not found, and so will not be deleted; - %return; -%end; - -%local fname1 fname2; -%let fname1=%mf_getuniquefileref(); -%let fname2=%mf_getuniquefileref(); - -filename &fname1 temp lrecl=10000; -filename &fname2 temp lrecl=10000; -data _null_ ; - file &fname1 ; - put ""; - put "SAS268436480"; - put ""; -run ; -proc metadata in=&fname1 out=&fname2 verbose;run; - -/* list the result */ -data _null_;infile &fname2; input; list; run; - -filename &fname1 clear; -filename &fname2 clear; - -/** - * Check deletion - */ -%local isgone; -data _null_; - length type uri $256; - call missing (of _all_); - rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri); - call symputx('isgone',type,'l'); -run; - -%mp_abort(iftrue=(&isgone = SASLibrary) - ,mac=&sysmacroname - ,msg=%str(Library (&name) NOT deleted) -) - -%put &sysmacroname: Library &name (&liburi) was successfully deleted; - -%mend mm_deletelibrary; diff --git a/003_macros/mm_deletestp.sas b/003_macros/mm_deletestp.sas deleted file mode 100644 index fc3d2a4..0000000 --- a/003_macros/mm_deletestp.sas +++ /dev/null @@ -1,70 +0,0 @@ -/*** HELP START ***//** - @file mm_deletestp.sas - @brief Deletes a Stored Process using path as reference - @details Will only delete the metadata, not any physical files associated. - - Usage: - - %mm_deletestp(target=/some/meta/path/myStoredProcess) - -

SAS Macros

- - @param [in] target= full path to the STP being deleted - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_deletestp( - target= -)/*/STORE SOURCE*/; - -/** - * Check STP does exist - */ -%local cmtype; -data _null_; - length type uri $256; - rc=metadata_pathobj("","&target",'StoredProcess',type,uri); - call symputx('cmtype',type,'l'); - call symputx('stpuri',uri,'l'); -run; -%if &cmtype ne ClassifierMap %then %do; - %put NOTE: No Stored Process found at ⌖ - %return; -%end; - -filename __in temp lrecl=10000; -filename __out temp lrecl=10000; -data _null_ ; - file __in ; - put ""; - put "SAS268436480"; - put ""; -run ; -proc metadata in=__in out=__out verbose;run; - -/* list the result */ -data _null_;infile __out; input; list; run; - -filename __in clear; -filename __out clear; - -/** - * Check deletion - */ -%local isgone; -data _null_; - length type uri $256; - call missing (of _all_); - rc=metadata_pathobj("","&target",'Note',type,uri); - call symputx('isgone',type,'l'); -run; -%if &isgone = ClassifierMap %then %do; - %put %str(ERR)OR: STP not deleted from ⌖ - %let syscc=4; - %return; -%end; - -%mend mm_deletestp; diff --git a/003_macros/mm_getauthinfo.sas b/003_macros/mm_getauthinfo.sas deleted file mode 100644 index f0013d0..0000000 --- a/003_macros/mm_getauthinfo.sas +++ /dev/null @@ -1,119 +0,0 @@ -/*** HELP START ***//** - @file mm_getauthinfo.sas - @brief Extracts authentication info for each user in metadata - @details - Usage: - - %mm_getauthinfo(outds=auths) - - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs - @param [out] outds= (mm_getauthinfo) The output dataset to create - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mm_getdetails.sas - @li mm_getobjects.sas - - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getauthinfo(outds=mm_getauthinfo - ,mdebug=0 -)/*/STORE SOURCE*/; -%local prefix fileref; -%let prefix=%substr(%mf_getuniquename(),1,25); - -%mm_getobjects(type=Login,outds=&prefix.0) - -%local fileref; -%let fileref=%mf_getuniquefileref(); - -data _null_; - file &fileref; - set &prefix.0 end=last; - /* run macro */ - str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_ - ,",outassocs=&prefix.a",_n_,")"); - put str; - /* transpose attributes */ - str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da" - ,_n_,"(drop=_name_);var value;id name;run;"); - put str; - /* add extra info to attributes */ - str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id=" - ,quote(trim(id)),";set &prefix.da",_n_ - ,";login_name=trim(subpad(name,1,256));drop name;run;"); - put str; - /* add extra info to associations */ - str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id=" - ,quote(trim(id)),";login_name=",quote(trim(name)) - ,";set &prefix.a",_n_,";run;"); - put str; - if last then do; - /* collate attributes */ - str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;"); - put str; - /* collate associations */ - str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;"); - put str; - /* tidy up */ - str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;"); - put str; - str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;"); - put str; - str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;"); - put str; - end; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fileref; - if _n_=1 then putlog // "Now executing the following code:" //; - input; putlog _infile_; - run; -%end; -%inc &fileref; -filename &fileref clear; - -/* get libraries */ -proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp; - by login_id; -data &prefix._temp; - set &prefix._temp; - by login_id; - length library_list $32767; - retain library_list; - if first.login_id then library_list=name; - else library_list=catx(' !! ',library_list,name); -proc sql; -/* get auth domain */ -create table &prefix._dom as - select login_id,name as domain - from &prefix._logas - where assoc='Domain'; -create unique index login_id on &prefix._dom(login_id); -/* join it all together */ -create table &outds as - select a.* - ,c.domain - ,b.library_list - from &prefix._logat (drop=ishidden lockedby usageversion publictype) a - left join &prefix._temp b - on a.login_id=b.login_id - left join &prefix._dom c - on a.login_id=c.login_id; - -%if &mdebug=0 %then %do; - proc datasets lib=work; - delete &prefix:; - run; -%end; - - -%mend mm_getauthinfo; diff --git a/003_macros/mm_getcols.sas b/003_macros/mm_getcols.sas deleted file mode 100644 index 3a1886b..0000000 --- a/003_macros/mm_getcols.sas +++ /dev/null @@ -1,53 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all metadata columns for a particular table - @details - - usage: - - %mm_getcols(tableuri=A5X8AHW1.B40001S5) - - @param [out] outds the dataset to create that contains the list of columns - @param [in] uri the uri of the table for which to return columns - - @returns outds dataset containing all columns, specifically: - - colname - - coluri - - coldesc - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getcols( - tableuri= - ,outds=work.mm_getcols -)/*/STORE SOURCE*/; - -data &outds; - keep col: SAS:; - length assoc uri coluri colname coldesc SASColumnType SASFormat SASInformat - SASPrecision SASColumnLength $256; - call missing (of _all_); - uri=symget('tableuri'); - n=1; - do while (metadata_getnasn(uri,'Columns',n,coluri)>0); - rc3=metadata_getattr(coluri,"Name",colname); - rc3=metadata_getattr(coluri,"Desc",coldesc); - rc4=metadata_getattr(coluri,"SASColumnType",SASColumnType); - rc5=metadata_getattr(coluri,"SASFormat",SASFormat); - rc6=metadata_getattr(coluri,"SASInformat",SASInformat); - rc7=metadata_getattr(coluri,"SASPrecision",SASPrecision); - rc8=metadata_getattr(coluri,"SASColumnLength",SASColumnLength); - output; - call missing(colname,coldesc,SASColumnType,SASFormat,SASInformat - ,SASPrecision,SASColumnLength); - n+1; - end; -run; -proc sort; - by colname; -run; - -%mend mm_getcols; diff --git a/003_macros/mm_getdetails.sas b/003_macros/mm_getdetails.sas deleted file mode 100644 index 02b3937..0000000 --- a/003_macros/mm_getdetails.sas +++ /dev/null @@ -1,68 +0,0 @@ -/*** HELP START ***//** - @file mm_getdetails.sas - @brief extracts metadata attributes and associations for a particular uri - - @param [in] uri the metadata object for which to return - attributes / associations - @param [out] outattrs= (work.attributes) - The dataset to create that contains the list of attributes - @param [out] outassocs= (work.associations) - The dataset to contain the list of associations - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getdetails(uri - ,outattrs=work.attributes - ,outassocs=work.associations -)/*/STORE SOURCE*/; - -data &outassocs; - keep assoc assocuri name; - length assoc assocuri name $256; - call missing(of _all_); - rc1=1;n1=1; - do while(rc1>0); - /* Walk through all possible associations of this object. */ - rc1=metadata_getnasl("&uri",n1,assoc); - rc2=1;n2=1; - do while(rc2>0); - /* Walk through all the associations on this machine object. */ - rc2=metadata_getnasn("&uri",trim(assoc),n2,assocuri); - if (rc2>0) then do; - rc3=metadata_getattr(assocuri,"Name",name); - output; - end; - call missing(name,assocuri); - n2+1; - end; - n1+1; - end; -run; -proc sort; - by assoc name; -run; - -data &outattrs; - keep type name value; - length type $4 name $256 value $32767; - rc1=1;n1=1;type='Prop';name='';value=''; - do while(rc1>0); - rc1=metadata_getnprp("&uri",n1,name,value); - if rc1>0 then output; - n1+1; - end; - rc1=1;n1=1;type='Attr'; - do while(rc1>0); - rc1=metadata_getnatr("&uri",n1,name,value); - if rc1>0 then output; - n1+1; - end; -run; -proc sort; - by type name; -run; - -%mend mm_getdetails; diff --git a/003_macros/mm_getdirectories.sas b/003_macros/mm_getdirectories.sas deleted file mode 100644 index 947a6b4..0000000 --- a/003_macros/mm_getdirectories.sas +++ /dev/null @@ -1,58 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a dataset with the meta directory object for a physical path - @details Provide a file path to get matching directory objects, or leave - blank to return all directories. The Directory object is used to reference - a physical filepath (eg when registering a .sas program in a Stored process) - - @param [in] path= Physical path for which to return a meta Directory object - @param [out] outds= (work.mm_getdirectories) - the dataset to create that contains the list of directories - @param [in] mDebug= (0) set to 1 to show debug messages in the log - - @returns outds dataset containing the following columns: - - directoryuri - - groupname - - groupdesc - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getDirectories( - path= - ,outds=work.mm_getDirectories - ,mDebug=0 -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_getDirectories.sas; -%&mD.put _local_; - -data &outds (keep=directoryuri name directoryname directorydesc ); - length directoryuri name directoryname directorydesc $256; - call missing(of _all_); - __i+1; -%if %length(&path)=0 %then %do; - do while - (metadata_getnobj("omsobj:Directory?@Id contains '.'",__i,directoryuri)>0); -%end; %else %do; - do while( - metadata_getnobj("omsobj:Directory?@DirectoryName='&path'",__i,directoryuri) - >0 - ); -%end; - __rc1=metadata_getattr(directoryuri, "Name", name); - __rc2=metadata_getattr(directoryuri, "DirectoryName", directoryname); - __rc3=metadata_getattr(directoryuri, "Desc", directorydesc); - &mD.putlog (_all_) (=); - drop __:; - __i+1; - if sum(of __rc1-__rc3)=0 then output; - end; -run; - -%mend mm_getDirectories; diff --git a/003_macros/mm_getdocument.sas b/003_macros/mm_getdocument.sas deleted file mode 100644 index 3766f96..0000000 --- a/003_macros/mm_getdocument.sas +++ /dev/null @@ -1,146 +0,0 @@ -/*** HELP START ***//** - @file - @brief Writes the TextStore of a Document Object to an external file - @details If the document exists, and has a textstore object, the contents - of that textstore are written to an external file. - - usage: - - %mm_getdocument(tree=/some/meta/path - ,name=someDocument - ,outref=/some/unquoted/filename.ext - ) - -

SAS Macros

- @li mp_abort.sas - - @param [in] tree= The metadata path of the document - @param [in] name= Document object name. - @param [out] outref= full and unquoted path to the desired text file. - This will be overwritten if it already exists. - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getdocument( - tree=/User Folders/sasdemo - ,name=myNote - ,outref=%sysfunc(pathname(work))/mm_getdocument.txt - ,mDebug=1 - ); - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -/** - * check tree exists - */ - -data _null_; - length type uri $256; - rc=metadata_pathobj("","&tree","Folder",type,uri); - call symputx('type',type,'l'); - call symputx('treeuri',uri,'l'); -run; - -%mp_abort( - iftrue= (&type ne Tree) - ,mac=mm_getdocument.sas - ,msg=Tree &tree does not exist! -) - -/** - * Check object exists - */ -data _null_; - length type docuri tsuri tsid $256 ; - rc1=metadata_pathobj("","&tree/&name","Note",type,docuri); - rc2=metadata_getnasn(docuri,"Notes",1,tsuri); - rc3=metadata_getattr(tsuri,"Id",tsid); - call symputx('type',type,'l'); - call symputx("tsid",tsid,'l'); - putlog (_all_)(=); -run; - -%mp_abort( - iftrue= (&type ne Document) - ,mac=mm_getdocument.sas - ,msg=Document &name could not be found in &tree! -) - -/** - * Now we can extract the textstore - */ -filename __getdoc temp lrecl=10000000; -proc metadata - in="$METAREPOSITORY - - SAS1" - out=__getdoc ; -run; - -/* find the beginning of the text */ -data _null_; - infile __getdoc lrecl=10000; - input; - start=index(_infile_,'StoredText="'); - if start then do; - call symputx("start",start+11); - put start= "type=&type"; - putlog '"' _infile_ '"'; - end; - stop; - -/* read the content, byte by byte, resolving escaped chars */ -filename __outdoc "&outref" lrecl=100000; -data _null_; - length filein 8 fileid 8; - filein = fopen("__getdoc","I",1,"B"); - fileid = fopen("__outdoc","O",1,"B"); - rec = "20"x; - length entity $6; - do while(fread(filein)=0); - x+1; - if x>&start then do; - rc = fget(filein,rec,1); - if rec='"' then leave; - else if rec="&" then do; - entity=rec; - do until (rec=";"); - if fread(filein) ne 0 then goto getout; - rc = fget(filein,rec,1); - entity=cats(entity,rec); - end; - select (entity); - when ('&' ) rec='&' ; - when ('<' ) rec='<' ; - when ('>' ) rec='>' ; - when (''') rec="'" ; - when ('"') rec='"' ; - when (' ') rec='0A'x; - when (' ') rec='0D'x; - when ('$' ) rec='$' ; - when (' ') rec='09'x; - otherwise putlog "%str(WARN)ING: missing value for " entity=; - end; - rc =fput(fileid, substr(rec,1,1)); - rc =fwrite(fileid); - end; - else do; - rc =fput(fileid,rec); - rc =fwrite(fileid); - end; - end; - end; - getout: - rc=fclose(filein); - rc=fclose(fileid); -run; -filename __getdoc clear; -filename __outdoc clear; - -%mend mm_getdocument; diff --git a/003_macros/mm_getfoldermembers.sas b/003_macros/mm_getfoldermembers.sas deleted file mode 100644 index 4c6a1b6..0000000 --- a/003_macros/mm_getfoldermembers.sas +++ /dev/null @@ -1,96 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns all direct child members of a particular folder - @details Displays the children for a particular folder, in a similar fashion - to the viya counterpart (mv_getfoldermembers.sas) - - Usage: - - %mm_getfoldermembers(root=/, outds=rootfolders) - - %mm_getfoldermembers(root=/User Folders/&sysuserid, outds=usercontent) - - @param [in] root= the parent folder under which to return all contents - @param [out] outds= the dataset to create that contains the list of - directories - @param [in] mDebug= set to 1 to show debug messages in the log - -

Data Outputs

- - Example for `root=/`: - - |metauri $17|metaname $256|metatype $32| - |---|---|---| - |A5XLSNXI.AA000001|Products |Folder| - |A5XLSNXI.AA000002|Shared Data |Folder| - |A5XLSNXI.AA000003|User Folders |Folder| - |A5XLSNXI.AA000004|System |Folder| - |A5XLSNXI.AA00003K|30.SASApps |Folder| - |A5XLSNXI.AA00006A|Public|Folder| - -

SAS Macros

- @li mm_getfoldertree.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ -%macro mm_getfoldermembers( - root= - ,outds=work.mm_getfoldertree -)/*/STORE SOURCE*/; - -%if "&root" = "/" %then %do; - %local fname1 fname2 fname3; - %let fname1=%mf_getuniquefileref(); - %let fname2=%mf_getuniquefileref(); - %let fname3=%mf_getuniquefileref(); - data _null_ ; - file &fname1 ; - put '' ; - put '$METAREPOSITORY' ; - put 'Tree' ; - put 'SAS' ; - put '388' ; - put '' ; - put ''; - put '' ; - put '' ; - run ; - proc metadata in=&fname1 out=&fname2 verbose;run; - - /* create an XML map to read the response */ - data _null_; - file &fname3; - put ''; - put ''; - put '//Objects/Tree'; - put '>17'; - put '//Objects/Tree/@Id'; - put '256>'; - put '//Objects/Tree/@Name'; - put '
'; - run; - %local libref1; - %let libref1=%mf_getuniquelibref(); - libname &libref1 xml xmlfileref=&fname2 xmlmap=&fname3; - - data &outds; - length metatype $32; - retain metatype 'Folder'; - set &libref1..sasfolders; - run; - -%end; -%else %do; - %mm_getfoldertree(root=&root, outds=&outds,depth=1) - data &outds; - set &outds(rename=(name=metaname publictype=metatype)); - keep metaname metauri metatype; - run; -%end; - -%mend mm_getfoldermembers; diff --git a/003_macros/mm_getfoldertree.sas b/003_macros/mm_getfoldertree.sas deleted file mode 100644 index c9c3a3c..0000000 --- a/003_macros/mm_getfoldertree.sas +++ /dev/null @@ -1,89 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns all folders / subfolder content for a particular root - @details Shows all members and SubTrees recursively for a particular root. - Note - for big sites, this returns a lot of data! So you may wish to reduce - the logging to speed up the process (see example below), OR - use mm_tree.sas - which uses proc metadata and is far more efficient. - - Usage: - - options ps=max nonotes nosource; - %mm_getfoldertree(root=/My/Meta/Path, outds=iwantthisdataset) - options notes source; - - @param [in] root= the parent folder under which to return all contents - @param [out] outds= the dataset to create that contains the list of - directories - @param [in] mDebug= set to 1 to show debug messages in the log - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ -%macro mm_getfoldertree( - root= - ,outds=work.mm_getfoldertree - ,mDebug=0 - ,depth=50 /* how many nested folders to query */ - ,level=1 /* system var - to track current level depth */ - ,append=NO /* system var - when YES means appending within nested loop */ -)/*/STORE SOURCE*/; - -%if &level>&depth %then %return; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname; -%&mD.put _local_; - -%if &append=NO %then %do; - /* ensure table doesn't exist already */ - data &outds; run; - proc sql; drop table &outds; -%end; - -/* get folder contents */ -data &outds.TMP/view=&outds.TMP; - length metauri pathuri $64 name $256 path $1024 - assoctype publictype MetadataUpdated MetadataCreated $32; - keep metauri assoctype name publictype MetadataUpdated MetadataCreated path; - call missing(of _all_); - path="&root"; - rc=metadata_pathobj("",path,"Folder",publictype,pathuri); - if publictype ne 'Tree' then do; - putlog "%str(WAR)NING: Tree " path 'does not exist!' publictype=; - stop; - end; - __n1=1; - do while(metadata_getnasl(pathuri,__n1,assoctype)>0); - __n1+1; - /* Walk through all possible associations of this object. */ - __n2=1; - if assoctype in ('Members','SubTrees') then - do while(metadata_getnasn(pathuri,assoctype,__n2,metauri)>0); - __n2+1; - call missing(name,publictype,MetadataUpdated,MetadataCreated); - __rc1=metadata_getattr(metauri,"Name", name); - __rc2=metadata_getattr(metauri,"MetadataUpdated", MetadataUpdated); - __rc3=metadata_getattr(metauri,"MetadataCreated", MetadataCreated); - __rc4=metadata_getattr(metauri,"PublicType", PublicType); - output; - end; - n1+1; - end; - drop __:; -run; - -proc append base=&outds data=&outds.TMP; -run; - -data _null_; - set &outds.TMP(where=(assoctype='SubTrees')); - call execute('%mm_getfoldertree(root=' - !!cats(path,"/",name)!!",outds=&outds,mDebug=&mdebug,depth=&depth" - !!",level=%eval(&level+1),append=YES)"); -run; - -%mend mm_getfoldertree; diff --git a/003_macros/mm_getgroupmembers.sas b/003_macros/mm_getgroupmembers.sas deleted file mode 100644 index 821df50..0000000 --- a/003_macros/mm_getgroupmembers.sas +++ /dev/null @@ -1,79 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates dataset with all members of a metadata group - @details This macro will query SAS metadata and return all the members - of a particular group. - - Usage: - - %mm_getgroupmembers(someGroupName - ,outds=work.mm_getgroupmembers - ,emails=YES - ) - - @param [in] group metadata group for which to bring back members - @param [out] outds= (work.mm_getgroupmembers) - The dataset to create that contains the list of members - @param [in] emails= (NO) Set to YES to bring back email addresses - @param [in] id= (NO) Set to yes if passing an ID rather than a group name - - @returns outds dataset containing all members of the metadata group - -

Related Macros

- @li mm_getgorups.sas - @li mm_adduser2group.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getgroupmembers( - group /* metadata group for which to bring back members */ - ,outds=work.mm_getgroupmembers /* output dataset to contain the results */ - ,emails=NO /* set to yes to bring back emails also */ - ,id=NO /* set to yes if passing an ID rather than group name */ -)/*/STORE SOURCE*/; - - data &outds ; - attrib uriGrp uriMem GroupId GroupName Group_or_Role MemberName MemberType - euri email length=$64 - GroupDesc length=$256 - rcGrp rcMem rc i j length=3; - call missing (of _all_); - drop uriGrp uriMem rcGrp rcMem rc i j arc ; - - i=1; - * Grab the URI for the first Group ; - %if &id=NO %then %do; - rcGrp=metadata_getnobj("omsobj:IdentityGroup?@Name='&group'",i,uriGrp); - %end; - %else %do; - rcGrp=metadata_getnobj("omsobj:IdentityGroup?@Id='&group'",i,uriGrp); - %end; - * If Group found, enter do loop ; - if rcGrp>0 then do; - call missing (rcMem,uriMem,GroupId,GroupName,Group_or_Role - ,MemberName,MemberType); - * get group info ; - rc = metadata_getattr(uriGrp,"Id",GroupId); - rc = metadata_getattr(uriGrp,"Name",GroupName); - rc = metadata_getattr(uriGrp,"PublicType",Group_or_Role); - rc = metadata_getattr(uriGrp,"Desc",GroupDesc); - j=1; - do while (metadata_getnasn(uriGrp,"MemberIdentities",j,uriMem) > 0); - call missing (MemberName, MemberType, email); - rc = metadata_getattr(uriMem,"Name",MemberName); - rc = metadata_getattr(uriMem,"PublicType",MemberType); - if membertype='User' and "&emails"='YES' then do; - if metadata_getnasn(uriMem,"EmailAddresses",1,euri)>0 then do; - arc=metadata_getattr(euri,"Address",email); - end; - end; - output; - j+1; - end; - end; - run; - -%mend mm_getgroupmembers; diff --git a/003_macros/mm_getlibmetadiffs.sas b/003_macros/mm_getlibmetadiffs.sas deleted file mode 100644 index 33ce5af..0000000 --- a/003_macros/mm_getlibmetadiffs.sas +++ /dev/null @@ -1,128 +0,0 @@ -/*** HELP START ***//** - @file - @brief Compares the metadata of a library with the physical tables - @details Creates a series of output tables that show the differences between - metadata and physical tables. - Each output can be created with an optional prefix. - - Credit - Paul Homes - https://platformadmin.com/blogs/paul/2012/11/sas-proc-metalib-ods-output - - Usage: - - %* create (and assign) a library for testing purposes ; - %mm_createlibrary( - libname=My Temp Library, - libref=XXTEMPXX, - tree=/User Folders/&sysuserid, - directory=%sysfunc(pathname(work)) - ) - - %* create some tables; - data work.table1 table2 table3; - a=1;b='two';c=3; - run; - - %* register the tables; - proc metalib; - omr=(library="My Temp Library"); - report(type=detail); - update_rule (delete); - run; - - %* modify the tables; - proc sql; - drop table table3; - alter table table2 drop c; - alter table table2 add d num; - - %* run the macro; - %mm_getlibmetadiffs(libname=My Temp Library) - - %* delete the library ; - %mm_deletelibrary(name=My Temp Library) - - The program will create four output tables, with the following structure (and - example data): - - #### &prefix.added - |name:$32.|metaID:$17.|SAStabName:$32.| - |---|---|---| - | | |DATA1| - - #### &prefix.deleted - |name:$32.|metaID:$17.|SAStabName:$32.| - |---|---|---| - |TABLE3|A5XLSNXI.BK0001HO|TABLE3| - - #### &prefix.updated - |tabName:$32.|tabMetaID:$17.|SAStabName:$32.|metaName:$32.|metaID:$17.|sasname:$32.|metaType:$16.|change:$64.| - |---|---|---|---|---|---|---|---| - |TABLE2|A5XLSNXI.BK0001HN|TABLE2|c|A5XLSNXI.BM000MA9|c|Column|Deleted| - | | | |d| |d|Column|Added| - - #### &prefix.meta - |Label1:$28.|cValue1:$1.|nValue1:D12.3| - |---|---|---| - |Total tables analyzed|4|4| - |Tables to be Updated|1|1| - |Tables to be Deleted|1|1| - |Tables to be Added|1|1| - |Tables matching data source|1|1| - |Tables not processed|0|0| - - If you are interested in more functionality like this (checking the health of - SAS metadata and your SAS 9 environment) then do contact [Allan Bowe]( - https://www.linkedin.com/in/allanbowe) for details of our SAS 9 Health Check - service. - - Our system scan will perform hundreds of checks to identify common issues, - such as dangling metadata, embedded passwords, security issues and more. - - @param [in] libname= the metadata name of the library to be compared - @param [out] outlib=(work) The library in which to store the output tables. - @param [out] prefix=(metadiff) The prefix for the four tables created. - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getlibmetadiffs( - libname= , - prefix=metadiff, - outlib=work -)/*/STORE SOURCE*/; - - /* create tempds */ - data;run; - %local tempds; - %let tempds=&syslast; - - /* save options */ - proc optsave out=&tempds; - run; - - options VALIDVARNAME=ANY VALIDMEMNAME=EXTEND; - - ods output - factoid1=&outlib..&prefix.meta - updtab=&outlib..&prefix.updated - addtab=&outlib..&prefix.added - deltab=&outlib..&prefix.deleted - ; - - proc metalib; - omr=(library="&libname"); - noexec; - report(type=detail); - update_rule (delete); - run; - - ods output close; - - /* restore options */ - proc optload data=&tempds; - run; - -%mend mm_getlibmetadiffs; diff --git a/003_macros/mm_getlibs.sas b/003_macros/mm_getlibs.sas deleted file mode 100644 index a6baa84..0000000 --- a/003_macros/mm_getlibs.sas +++ /dev/null @@ -1,97 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all metadata libraries - @details Will only show the libraries to which a user has the requisite - metadata access. - - @param [out] outds= (work.mm_getlibs) - The library.dataset to create that contains the list of libraries - - @returns outds dataset containing all groups in a column named "metagroup" - (defaults to work.mm_getlibs). The following columns are provided: - - LibraryId - - LibraryName - - LibraryRef - - Engine - - @warning The following filenames are created and then de-assigned: - - filename sxlemap clear; - filename response clear; - libname _XML_ clear; - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getlibs( - outds=work.mm_getLibs -)/*/STORE SOURCE*/; - -/* - flags: - - OMI_SUCCINCT (2048) Do not return attributes with null values. - OMI_GET_METADATA (256) Executes a GetMetadata call for each object that - is returned by the GetMetadataObjects method. - OMI_ALL_SIMPLE (8) Gets all of the attributes of the requested object. -*/ -data _null_; - flags=2048+256+8; - call symputx('flags',flags,'l'); -run; - -* use a temporary fileref to hold the response; -filename response temp; -/* get list of libraries */ -proc metadata in= - ' - $METAREPOSITORY - SASLibrary - - SAS - &flags - - ' - out=response; -run; - -/* write the response to the log for debugging */ -data _null_; - infile response lrecl=32767; - input; - put _infile_; -run; - -/* create an XML map to read the response */ -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put ''; - put '//Objects/SASLibrary'; - put '>17'; - put '//Objects/SASLibrary/@Id'; - put '256>'; - put '//Objects/SASLibrary/@Name'; - put '8'; - put '//Objects/SASLibrary/@Libref'; - put '>12'; - put '//Objects/SASLibrary/@Engine'; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -/* sort the response by library name */ -proc sort data=_XML_.saslibrary out=&outds; - by libraryname; -run; - - -/* clear references */ -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_getlibs; diff --git a/003_macros/mm_getobjects.sas b/003_macros/mm_getobjects.sas deleted file mode 100644 index 3da3393..0000000 --- a/003_macros/mm_getobjects.sas +++ /dev/null @@ -1,70 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all metadata objects for a particular type - - @param [in] type= the metadata type for which to return all objects - @param [out] outds= the dataset to create that contains the list of types - - @returns outds dataset containing all objects - - @warning The following filenames are created and then de-assigned: - - filename sxlemap clear; - filename response clear; - libname _XML_ clear; - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getobjects( - type=SASLibrary - ,outds=work.mm_getobjects -)/*/STORE SOURCE*/; - - -* use a temporary fileref to hold the response; -filename response temp; -/* get list of libraries */ -proc metadata in= - "$METAREPOSITORY - &typeSAS - 0" - out=response; -run; - -/* write the response to the log for debugging */ -data _null_; - infile response lrecl=1048576; - input; - put _infile_; -run; - -/* create an XML map to read the response */ -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put "/GetMetadataObjects/Objects/&type"; - put ""; - put ''; - put "/GetMetadataObjects/Objects/&type/@Id"; - put "characterstring200"; - put ''; - put "/GetMetadataObjects/Objects/&type/@Name"; - put "characterstring200"; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -proc sort data= _XML_.SASObjects out=&outds; - by name; -run; - -/* clear references */ -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_getobjects; diff --git a/003_macros/mm_getpublictypes.sas b/003_macros/mm_getpublictypes.sas deleted file mode 100644 index a06d256..0000000 --- a/003_macros/mm_getpublictypes.sas +++ /dev/null @@ -1,88 +0,0 @@ -/*** HELP START ***//** - @file mm_getpublictypes.sas - @brief Creates a dataset with all deployable public types - @details More info: - https://support.sas.com/documentation/cdl/en/bisag/65422/HTML/default/viewer.htm#n1nkrdzsq5iunln18bk2236istkb.htm - - Usage: - - * dataset will contain one column - publictype ($64); - %mm_getpublictypes(outds=types) - - @param [out] outds= (work.mm_getpublictypes) The library.dataset to create - - @returns outds= dataset containing all types - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getpublictypes( - outds=work.mm_getpublictypes -)/*/STORE SOURCE*/; - -proc sql; -create table &outds (publictype char(64)); /* longest is currently 52 */ -insert into &outds values ('ACT'); -insert into &outds values ('Action'); -insert into &outds values ('Application'); -insert into &outds values ('ApplicationServer'); -insert into &outds values ('BurstDefinition'); -insert into &outds values ('Channel'); -insert into &outds values ('Condition'); -insert into &outds values ('ConditionActionSet'); -insert into &outds values ('ContentSubscriber'); -insert into &outds values ('Cube'); -insert into &outds values ('DataExploration'); -insert into &outds values ('DeployedFlow'); -insert into &outds values ('DeployedJob'); -insert into &outds values ('Document'); -insert into &outds values ('EventSubscriber'); -insert into &outds values ('ExternalFile'); -insert into &outds values ('FavoritesFolder'); -insert into &outds values ('Folder'); -insert into &outds values ('Folder.SecuredData'); -insert into &outds values ('GeneratedTransform'); -insert into &outds values ('InformationMap'); -insert into &outds values ('InformationMap.OLAP'); -insert into &outds values ('InformationMap.Relational'); -insert into &outds values ('JMSDestination (Java Messaging System message queue)'); -insert into &outds values ('Job'); -insert into &outds values ('Job.Cube'); -insert into &outds values ('Library'); -insert into &outds values ('MessageQueue'); -insert into &outds values ('MiningResults'); -insert into &outds values ('MQM.JMS (queue manager for Java Messaging Service)'); -insert into &outds values ('MQM.MSMQ (queue manager for MSMQ)'); -insert into &outds values ('MQM.Websphere (queue manager for WebSphere MQ)'); -insert into &outds values ('Note'); -insert into &outds values ('OLAPSchema'); -insert into &outds values ('Project'); -insert into &outds values ('Project.EG'); -insert into &outds values ('Project.AMOExcel'); -insert into &outds values ('Project.AMOPowerPoint'); -insert into &outds values ('Project.AMOWord'); -insert into &outds values ('Prompt'); -insert into &outds values ('PromptGroup'); -insert into &outds values ('Report'); -insert into &outds values ('Report.Component'); -insert into &outds values ('Report.Image'); -insert into &outds values ('Report.StoredProcess'); -insert into &outds values ('Role'); -insert into &outds values ('SearchFolder'); -insert into &outds values ('SecuredLibrary'); -insert into &outds values ('Server'); -insert into &outds values ('Service.SoapGenerated'); -insert into &outds values ('SharedDimension'); -insert into &outds values ('Spawner.Connect'); -insert into &outds values ('Spawner.IOM (object spawner)'); -insert into &outds values ('StoredProcess'); -insert into &outds values ('SubscriberGroup.Content'); -insert into &outds values ('SubscriberGroup.Event'); -insert into &outds values ('Table'); -insert into &outds values ('User'); -insert into &outds values ('UserGroup'); -quit; - -%mend mm_getpublictypes; diff --git a/003_macros/mm_getrepos.sas b/003_macros/mm_getrepos.sas deleted file mode 100644 index 470bb83..0000000 --- a/003_macros/mm_getrepos.sas +++ /dev/null @@ -1,133 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all available repositories - - @param [out] outds= (work.mm_getrepos) - The dataset to create that contains the list of repos - - @returns outds dataset containing all repositories - - @warning The following filenames are created and then de-assigned: - - filename sxlemap clear; - filename response clear; - libname _XML_ clear; - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getrepos( - outds=work.mm_getrepos -)/*/STORE SOURCE*/; - - -* use a temporary fileref to hold the response; -filename response temp; -/* get list of libraries */ -proc metadata in= - "1" - out=response; -run; - -/* write the response to the log for debugging */ -/* -data _null_; - infile response lrecl=1048576; - input; - put _infile_; -run; -*/ - -/* create an XML map to read the response */ -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put "/GetRepositories/Repositories/Repository"; - put ""; - put ''; - put "/GetRepositories/Repositories/Repository/@Id"; - put ""; - put "characterstring200"; - put ''; - put ''; - put "/GetRepositories/Repositories/Repository/@Name"; - put ""; - put "characterstring200"; - put ''; - put ''; - put "/GetRepositories/Repositories/Repository/@Desc"; - put ""; - put "characterstring200"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@DefaultNS"; - put "characterstring200"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@RepositoryType"; - put "characterstring20"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@RepositoryFormat"; - put "characterstring10"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@Access"; - put "characterstring16"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@CurrentAccess"; - put "characterstring16"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@PauseState"; - put "characterstring16"; - put ''; - put ''; - put "/GetRepositories/Repositories/Repository/@Path"; - put ""; - put "characterstring256"; - put ''; - put ''; - put "/GetRepositories/Repositories/Repository/@Engine"; - put ""; - put "characterstring8"; - put ''; - put ''; - put "/GetRepositories/Repositories/Repository/@Options"; - put ""; - put "characterstring32"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@MetadataCreated"; - put "characterstring24"; - put ''; - put ''; - put ""; - put "/GetRepositories/Repositories/Repository/@MetadataUpdated"; - put "characterstring24"; - put ''; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -proc sort data= _XML_.SASRepos out=&outds; - by name; -run; - -/* clear references */ -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_getrepos; diff --git a/003_macros/mm_getroles.sas b/003_macros/mm_getroles.sas deleted file mode 100644 index 99f0362..0000000 --- a/003_macros/mm_getroles.sas +++ /dev/null @@ -1,74 +0,0 @@ -/*** HELP START ***//** - @file mm_getroles.sas - @brief Creates a table containing a list of roles - @details - - Usage: - - %mm_getroles() - - @param [out] outds= (work.mm_getroles) - The dataset to create that contains the list of roles - - @returns outds dataset containing all roles, with the following columns: - - uri - - name - - @warning The following filenames are created and then de-assigned: - - filename sxlemap clear; - filename response clear; - libname _XML_ clear; - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getroles( - outds=work.mm_getroles -)/*/STORE SOURCE*/; - -filename response temp; -options noquotelenmax; -proc metadata in= '$METAREPOSITORY - IdentityGroupSAS388 - - - - - ' - out=response; -run; - -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put "/GetMetadataObjects/Objects/IdentityGroup"; - put ""; - put ''; - put "/GetMetadataObjects/Objects/IdentityGroup/@Id"; - put ""; - put "characterstring32"; - put ''; - put "/GetMetadataObjects/Objects/IdentityGroup/@Name"; - put ""; - put "characterstring256"; - put ''; - put "/GetMetadataObjects/Objects/IdentityGroup/@Desc"; - put ""; - put "characterstring500"; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -proc sort data= _XML_.roles out=&outds; - by rolename; -run; - -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_getroles; diff --git a/003_macros/mm_getservercontexts.sas b/003_macros/mm_getservercontexts.sas deleted file mode 100644 index c47dfe2..0000000 --- a/003_macros/mm_getservercontexts.sas +++ /dev/null @@ -1,92 +0,0 @@ -/*** HELP START ***//** - @file mm_getservercontexts.sas - @brief Creates a dataset with all server contexts in all repos - @details - Usage: - - %mm_getservercontexts(outds=mm_getservercontexts) - - @param [out] outds= (work.mm_getrepos) - the dataset to create that contains the list - - @warning The following filenames are created and then de-assigned: - - filename __mc1 clear; - filename __mc2 clear; - libname __mc3 clear; - -

SAS Macros

- @li mm_getrepos.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getservercontexts( - outds=work.mm_getrepos -)/*/STORE SOURCE*/; -%local repo repocnt x; -%let repo=%sysfunc(getoption(metarepository)); - -/* first get list of available repos */ -%mm_getrepos(outds=work.repos) -%let repocnt=0; -data _null_; - set repos; - where repositorytype in('CUSTOM','FOUNDATION'); - keep id name ; - call symputx(cats('repo',_n_),name,'l'); - call symputx('repocnt',_n_,'l'); -run; - -filename __mc1 temp; -filename __mc2 temp; -data &outds; - length serveruri servername $200; - call missing (of _all_); - stop; -run; -%do x=1 %to &repocnt; - options metarepository=&&repo&x; - proc metadata in= - "$METAREPOSITORY - ServerContextSAS - 0" - out=__mc1; - run; - /* - data _null_; - infile __mc1 lrecl=1048576; - input; - put _infile_; - run; - */ - data _null_; - file __mc2; - put ''; - put "/GetMetadataObjects/Objects/ServerContext"; - put ""; - put ''; - put "/GetMetadataObjects/Objects/ServerContext/@Id"; - put ""; - put "characterstring200"; - put ''; - put ''; - put "/GetMetadataObjects/Objects/ServerContext/@Name"; - put ""; - put "characterstring200"; - put ''; - put '
'; - run; - libname __mc3 xml xmlfileref=__mc1 xmlmap=__mc2; - proc append base=&outds data=__mc3.SASContexts;run; - libname __mc3 clear; -%end; - -options metarepository=&repo; - -filename __mc1 clear; -filename __mc2 clear; - -%mend mm_getservercontexts; diff --git a/003_macros/mm_getstpcode.sas b/003_macros/mm_getstpcode.sas deleted file mode 100644 index 8101fba..0000000 --- a/003_macros/mm_getstpcode.sas +++ /dev/null @@ -1,170 +0,0 @@ -/*** HELP START ***//** - @file - @brief Writes the code of an STP to an external file - @details Fetches the SAS code from a Stored Process where the code is stored - in metadata. - - Usage: - - %mm_getstpcode(tree=/some/meta/path - ,name=someSTP - ,outloc=/some/unquoted/filename.ext - ) - - @param [in] tree= The metadata path of the Stored Process (can also contain - name) - @param [in] name= Stored Process name. Leave blank if included above. - @param [out] outloc= (0) full and unquoted path to the desired text file. - This will be overwritten if it already exists. - @param [out] outref= (0) Fileref to which to write the code. - @param [out] showlog=(NO) Set to YES to print log to the window - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mp_abort.sas - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getstpcode( - tree=/User Folders/sasdemo/somestp - ,name= - ,outloc=0 - ,outref=0 - ,mDebug=1 - ,showlog=NO - ); - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -%if %length(&name)>0 %then %let name=/&name; - -/* first, check if STP exists */ -%local tsuri; -%let tsuri=stopifempty ; - -data _null_; - format type uri tsuri value $200.; - call missing (of _all_); - path="&tree&name(StoredProcess)"; - /* first, find the STP ID */ - if metadata_pathobj("",path,"StoredProcess",type,uri)>0 then do; - /* get sourcecode */ - cnt=1; - do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0); - rc=metadata_getattr(tsuri,"Name",value); - &mD.put tsuri= value=; - if value="SourceCode" then do; - /* found it! */ - rc=metadata_getattr(tsuri,"Id",value); - call symputx('tsuri',value,'l'); - stop; - end; - cnt+1; - end; - end; - else put (_all_)(=); -run; - -%mp_abort(iftrue= (&tsuri=stopifempty) - ,mac=mm_getstpcode - ,msg=%str(&tree&name.(StoredProcess) not found!) -) - -/** - * Now we can extract the textstore - */ -filename __getdoc temp lrecl=10000000; -proc metadata - in="$METAREPOSITORY - - SAS1" - out=__getdoc ; -run; - -/* find the beginning of the text */ -%local start; -data _null_; - infile __getdoc lrecl=10000; - input; - start=index(_infile_,'StoredText="'); - if start then do; - call symputx("start",start+11); - *putlog '"' _infile_ '"'; - end; - stop; - -%local outeng; -%if "&outloc"="0" %then %let outeng=TEMP; -%else %let outeng="&outloc"; -%local fref; -%if &outref=0 %then %let fref=%mf_getuniquefileref(); -%else %let fref=&outref; - -/* read the content, byte by byte, resolving escaped chars */ -filename &fref &outeng lrecl=100000; -data _null_; - length filein 8 fileid 8; - filein = fopen("__getdoc","I",1,"B"); - fileid = fopen("&fref","O",1,"B"); - rec = "20"x; - length entity $6; - do while(fread(filein)=0); - x+1; - if x>&start then do; - rc = fget(filein,rec,1); - if rec='"' then leave; - else if rec="&" then do; - entity=rec; - do until (rec=";"); - if fread(filein) ne 0 then goto getout; - rc = fget(filein,rec,1); - entity=cats(entity,rec); - end; - select (entity); - when ('&' ) rec='&' ; - when ('<' ) rec='<' ; - when ('>' ) rec='>' ; - when (''') rec="'" ; - when ('"') rec='"' ; - when (' ') rec='0A'x; - when (' ') rec='0D'x; - when ('$' ) rec='$' ; - when (' ') rec='09'x; - otherwise putlog "%str(WARN)ING: missing value for " entity=; - end; - rc =fput(fileid, substr(rec,1,1)); - rc =fwrite(fileid); - end; - else do; - rc =fput(fileid,rec); - rc =fwrite(fileid); - end; - end; - end; - getout: - rc=fclose(filein); - rc=fclose(fileid); -run; - -%if &showlog=YES %then %do; - data _null_; - infile &fref lrecl=32767 end=last; - input; - if _n_=1 then putlog '>>stpcodeBEGIN<<'; - putlog _infile_; - if last then putlog '>>stpcodeEND<<'; - run; -%end; - -filename __getdoc clear; -%if &outref=0 %then %do; - filename &fref clear; -%end; - -%mend mm_getstpcode; diff --git a/003_macros/mm_getstpinfo.sas b/003_macros/mm_getstpinfo.sas deleted file mode 100644 index 23a44f2..0000000 --- a/003_macros/mm_getstpinfo.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file - @brief Get the properties of a Stored Process - @details Extracts various properties and creates an output table in the - structure below: - -|STP_URI:$200.|SERVERCONTEXT:$200.|STOREDPROCESSCONFIGURATION:$1000.|SOURCECODE_FIRST32K:$32767.|PATH:$76.| -|---|---|---|---|---| -|`A5DN9TDQ.BH0000C8 `|`SASApp `|` `|`%put first 32767 bytes of code; `|`/path/to/my/stp`| - - @param [in] pgm The metadata path of the Stored Process - @param [out] outds= (work.mm_getstpinfo) The output table to create - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

Related Files

- @li mm_getstpcode.sas - @li mm_getstps.sas - @li mm_createstp.sas - @li mm_deletestp.sas - -**//*** HELP END ***/ - -%macro mm_getstpinfo(pgm - ,outds=work.mm_getstpinfo - ,mDebug=0 -); - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -data &outds; - length type stp_uri tsuri servercontext value $200 - StoredProcessConfiguration $1000 sourcecode_first32k $32767; - keep path stp_uri sourcecode_first32k StoredProcessConfiguration - servercontext; - call missing (of _all_); - path="&pgm(StoredProcess)"; - /* first, find the STP ID */ - if metadata_pathobj("",path,"StoredProcess",type,stp_uri)>0 then do; - /* get attributes */ - cnt=1; - do while (metadata_getnasn(stp_uri,"Notes",cnt,tsuri)>0); - rc1=metadata_getattr(tsuri,"Name",value); - &mD.put tsuri= value=; - if value="SourceCode" then do; - rc2=metadata_getattr(tsuri,"StoredText",sourcecode_first32k); - end; - else if value="Stored Process" then do; - rc3=metadata_getattr(tsuri,"StoredText",StoredProcessConfiguration); - end; - cnt+1; - end; - /* get context (should only be one) */ - rc4=metadata_getnasn(stp_uri,"ComputeLocations",1,tsuri); - rc5=metadata_getattr(tsuri,"Name",servercontext); - end; - else do; - put 'ERR' +(-1) "OR: could not find " path; - put (_all_)(=); - end; - &md.put (_all_)(=); -run; - -%mend mm_getstpinfo ; diff --git a/003_macros/mm_getstps.sas b/003_macros/mm_getstps.sas deleted file mode 100644 index b8d0f7c..0000000 --- a/003_macros/mm_getstps.sas +++ /dev/null @@ -1,110 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a dataset with all Stored Processes, or just those in a - particular folder / with a particular name. - @details Leave blank to get all stps. Provide a Tree (path or uri) or a - name (not case sensitive) to filter that way also. - usage: - - %mm_getstps() - - %mm_getstps(name=My STP) - - %mm_getstps(tree=/My Folder/My STPs) - - %mm_getstps(tree=/My Folder/My STPs, name=My STP) - -

SAS Macros

- @li mm_gettree.sas - - @param [in] tree= () the metadata folder location in which to search. - Leave blank for all folders. Does not search subdirectories. - @param [in] name= Provide the name of an STP to search for just that one. Can - combine with the tree= parameter. - @param [out] outds= the dataset to create that contains the list of stps. - @param [in] mDebug= set to 1 to show debug messages in the log - @param [in] showDesc= provide a non blank value to return stored process - descriptions - @param [in] showUsageVersion= () - Provide a non blank value to return the UsageVersion. - This is either 1000000 (type 1, 9.2) or 2000000 (type2, 9.3 onwards). - - @returns outds dataset containing the following columns - - stpuri - - stpname - - treeuri - - stpdesc (if requested) - - usageversion (if requested) - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getstps( - tree= - ,name= - ,outds=work.mm_getstps - ,mDebug=0 - ,showDesc= - ,showUsageVersion= -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_getstps.sas; -%&mD.put _local_; - -data &outds; - length stpuri stpname usageversion treeuri stpdesc $256; - call missing (of _all_); -run; - -%if %length(&tree)>0 %then %do; - /* get tree info */ - %mm_gettree(tree=&tree,inds=&outds, outds=&outds, mDebug=&mDebug) - %if %mf_nobs(&outds)=0 %then %do; - %put NOTE: Tree &tree did not exist!!; - %return; - %end; -%end; - - -data &outds ; - set &outds(rename=(treeuri=treeuri_compare)); - length treeuri query stpuri $256; - i+1; -%if %length(&name)>0 %then %do; - query="omsobj:ClassifierMap?@PublicType='StoredProcess' and @Name='&name'"; - putlog query=; -%end; -%else %do; - query="omsobj:ClassifierMap?@PublicType='StoredProcess'"; -%end; -%if &mDebug=1 %then %do; - putlog 'start' (_all_)(=); -%end; - do while(00 %then %do; - if treeuri ne treeuri_compare then goto exitloop; - %end; - %if %length(&showDesc)>0 %then %do; - rc3=metadata_getattr(stpuri,"Desc", stpdesc); - keep stpdesc; - %end; - %if %length(&showUsageVersion)>0 %then %do; - rc4=metadata_getattr(stpuri,"UsageVersion",UsageVersion); - keep usageversion; - %end; - output; - &mD.put (_all_)(=); - exitloop: - end; - keep stpuri stpname treeuri; -run; - -%mend mm_getstps; diff --git a/003_macros/mm_gettableid.sas b/003_macros/mm_gettableid.sas deleted file mode 100644 index a1450ff..0000000 --- a/003_macros/mm_gettableid.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file mm_gettableid.sas - @brief Get the metadata id for a particular table - @details Provide a libref and table name to return the corresponding metadata - in an output datataset. - - Usage: - - - get a table id - %mm_gettableid(libref=METALIB,ds=SOMETABLE,outds=iwant) - - @param [in] libref= The libref to search - @param [in] ds= The input dataset to check - @param [out] outds= the dataset to create that contains the `tableuri` - @param [in] mDebug= set to 1 to show debug messages in the log - - @returns outds dataset containing `tableuri` and `tablename` - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_gettableid( - libref= - ,ds= - ,outds=work.mm_gettableid - ,mDebug=0 -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing &sysmacroname..sas; -%&mD.put _local_; - -data &outds; - length uri usingpkguri id type tableuri tablename tmpuri $256; - call missing(of _all_); - keep tableuri tablename; - n=1; - rc=0; - if metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",n,uri)<1 then do; - put "Library &libref not found"; - stop; - end; - &mD.putlog "uri is " uri; - if metadata_getnasn(uri, "UsingPackages", 1, usingpkguri)>0 then do; - rc=metadata_resolve(usingpkguri,type,id); - &mD.putlog "Type is " type; - end; - - if type='DatabaseSchema' then tmpuri=usingpkguri; - else tmpuri=uri; - - t=1; - do while(metadata_getnasn(tmpuri, "Tables", t, tableuri)>0); - t+1; - rc= metadata_getattr(tableuri, "Name", tablename); - &mD.putlog "Table is " tablename; - if upcase(tablename)="%upcase(&ds)" then do; - output; - end; - end; -run; - -%mend mm_gettableid; diff --git a/003_macros/mm_gettables.sas b/003_macros/mm_gettables.sas deleted file mode 100644 index bd22c6b..0000000 --- a/003_macros/mm_gettables.sas +++ /dev/null @@ -1,120 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all metadata tables for a particular library - @details Will only show the tables for which the executing user has the - requisite metadata access. - - usage: - - %mm_gettables(uri=A5X8AHW1.B40001S5) - - @param [in] uri= the uri of the library for which to return tables - @param [out] outds= (work.mm_gettables) the dataset to contain the list of - tables - @param [in] getauth= (YES) Fetch the authdomain used in database connections. - Set to NO to improve runtimes in larger environments, as there can be a - performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` - call. - - @returns outds dataset containing all groups in a column named "metagroup" - (defaults to work.mm_getlibs). The following columns are provided: - - tablename - - tableuri - - libref - - libname - - libdesc - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_gettables( - uri= - ,outds=work.mm_gettables - ,getauth=YES -)/*/STORE SOURCE*/; - - -data &outds; - length uri serveruri conn_uri domainuri libname ServerContext AuthDomain - path_schema usingpkguri type tableuri $256 id $17 - libdesc $200 libref engine $8 IsDBMSLibname IsPreassigned $1 - tablename $50 /* metadata table names can be longer than $32 */ - ; - keep libname libdesc libref engine ServerContext path_schema AuthDomain - tableuri tablename IsPreassigned IsDBMSLibname id; - call missing (of _all_); - - uri=symget('uri'); - rc= metadata_getattr(uri, "Name", libname); - if rc <0 then do; - put 'The library is not defined in this metadata repository.'; - stop; - end; - rc= metadata_getattr(uri, "Desc", libdesc); - rc= metadata_getattr(uri, "Libref", libref); - rc= metadata_getattr(uri, "Engine", engine); - rc= metadata_getattr(uri, "IsDBMSLibname", IsDBMSLibname); - rc= metadata_getattr(uri, "IsPreassigned", IsPreassigned); - rc= metadata_getattr(uri, "Id", Id); - - /*** Get associated ServerContext ***/ - rc= metadata_getnasn(uri, "DeployedComponents", 1, serveruri); - if rc > 0 then rc2= metadata_getattr(serveruri, "Name", ServerContext); - else ServerContext=''; - - /*** If the library is a DBMS library, get the Authentication Domain - associated with the DBMS connection credentials ***/ - if IsDBMSLibname="1" and "&getauth"='YES' then do; - rc= metadata_getnasn(uri, "LibraryConnection", 1, conn_uri); - if rc>0 then do; - rc2= metadata_getnasn(conn_uri, "Domain", 1, domainuri); - if rc2>0 then rc3= metadata_getattr(domainuri, "Name", AuthDomain); - end; - end; - - /*** Get the path/database schema for this library ***/ - rc=metadata_getnasn(uri, "UsingPackages", 1, usingpkguri); - if rc>0 then do; - rc=metadata_resolve(usingpkguri,type,id); - if type='Directory' then - rc=metadata_getattr(usingpkguri, "DirectoryName", path_schema); - else if type='DatabaseSchema' then - rc=metadata_getattr(usingpkguri, "Name", path_schema); - else path_schema="unknown"; - end; - - /*** Get the tables associated with this library ***/ - /*** If DBMS, tables are associated with DatabaseSchema ***/ - if type='DatabaseSchema' then do; - t=1; - ntab=metadata_getnasn(usingpkguri, "Tables", t, tableuri); - if ntab>0 then do t=1 to ntab; - tableuri=''; - tablename=''; - ntab=metadata_getnasn(usingpkguri, "Tables", t, tableuri); - tabrc= metadata_getattr(tableuri, "Name", tablename); - output; - end; - else put 'Library ' libname ' has no tables registered'; - end; - else if type in ('Directory','SASLibrary') then do; - t=1; - ntab=metadata_getnasn(uri, "Tables", t, tableuri); - if ntab>0 then do t=1 to ntab; - tableuri=''; - tablename=''; - ntab=metadata_getnasn(uri, "Tables", t, tableuri); - tabrc= metadata_getattr(tableuri, "Name", tablename); - output; - end; - else put 'Library ' libname ' has no tables registered'; - end; -run; - -proc sort; -by tablename tableuri; -run; - -%mend mm_gettables; diff --git a/003_macros/mm_gettree.sas b/003_macros/mm_gettree.sas deleted file mode 100644 index 31984a6..0000000 --- a/003_macros/mm_gettree.sas +++ /dev/null @@ -1,68 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the metadata path and object from either the path or object - @details Provide a metadata BIP tree path, or the uri for the bottom level - folder, to obtain a dataset (&outds) containing both the path - and uri. - - Usage: - - %mm_getTree(tree=/User Folders/sasdemo) - - - @param [in] tree= the BIP Tree folder path or uri - @param [out] outds= the dataset to create that contains the tree path & uri - @param [in] inds= an optional input dataset to augment with treepath & treeuri - @param [in] mDebug= set to 1 to show debug messages in the log - - @returns outds dataset containing the following columns: - - treeuri - - treepath - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getTree( - tree= - ,inds= - ,outds=work.mm_getTree - ,mDebug=0 -)/*/STORE SOURCE*/; - -%local mD; -%if &mDebug=1 %then %let mD=; -%else %let mD=%str(*); -%&mD.put Executing mm_getTree.sas; -%&mD.put _local_; - -data &outds; - length treeuri __parenturi __type __name $256 treepath $512; -%if %length(&inds)>0 %then %do; - set &inds; -%end; - __rc1=metadata_resolve("&tree",__type,treeuri); - - if __type='Tree' then do; - __rc2=metadata_getattr(treeuri,"Name",__name); - treepath=cats('/',__name); - /* get parents */ - do while (metadata_getnasn(treeuri,"ParentTree",1,__parenturi)>0); - __rc3=metadata_getattr(__parenturi,"Name",__name); - treepath=cats('/',__name,treepath); - treeuri=__parenturi; - end; - treeuri="&tree"; - end; - else do; - __rc2=metadata_pathobj(' ',"&tree",'Folder',__type,treeuri); - treepath="&tree"; - end; - - &mD.put (_all_)(=); - drop __:; - if treeuri ne "" and treepath ne "" then output; - stop; -run; -%mend mm_getTree; diff --git a/003_macros/mm_gettypes.sas b/003_macros/mm_gettypes.sas deleted file mode 100644 index 43cd8d1..0000000 --- a/003_macros/mm_gettypes.sas +++ /dev/null @@ -1,76 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with all metadata types - @details Usage: - - %mm_gettypes(outds=types) - - @param [in] outds= (work.mm_gettypes) - The dataset to create that contains the list of types - @returns outds dataset containing all types - @warning The following filenames are created and then de-assigned: - - filename sxlemap clear; - filename response clear; - libname _XML_ clear; - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_gettypes( - outds=work.mm_gettypes -)/*/STORE SOURCE*/; - -* use a temporary fileref to hold the response; -filename response temp; -/* get list of libraries */ -proc metadata in= - ' - - SAS - - 2048 - - - $METAREPOSITORY - - ' - out=response; -run; - -/* write the response to the log for debugging */ -data _null_; - infile response lrecl=1048576; - input; - put _infile_; -run; - -/* create an XML map to read the response */ -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put '//GetTypes/Types/Type'; - put '64'; - put '//GetTypes/Types/Type/@Id'; - put '256'; - put '//GetTypes/Types/Type/@Desc'; - put ''; - put '//GetTypes/Types/Type/@HasSubtypes'; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; -/* sort the response by library name */ -proc sort data=_XML_.sastypes out=&outds; - by id; -run; - - -/* clear references */ -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_gettypes; diff --git a/003_macros/mm_getusers.sas b/003_macros/mm_getusers.sas deleted file mode 100644 index 682b979..0000000 --- a/003_macros/mm_getusers.sas +++ /dev/null @@ -1,96 +0,0 @@ -/*** HELP START ***//** - @file mm_getusers.sas - @brief Creates a table containing a list of all users - @details Only shows a limited number of attributes as some sites will have a - LOT of users. - - Usage: - - %mm_getusers() - - Optionally, filter for a user (useful to get the uri): - - %mm_getusers(user=&_metaperson) - - @returns outds dataset containing all users, with the following columns: - - uri - - name - - @param [in] user= (0) Set to a metadata user to filter on that user - @param [out] outds= (work.mm_getusers) The output table to create - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_getusers( - outds=work.mm_getusers, - user=0 -)/*/STORE SOURCE*/; - -filename response temp; -%if %superq(user)=0 %then %do; - proc metadata in= ' - $METAREPOSITORY - Person - SAS - 0 - - - - - - ' - out=response; - run; -%end; -%else %do; - filename inref temp; - data _null_; - file inref; - put ""; - put "$METAREPOSITORY"; - put "Person"; - put "SAS"; - put ""; - put "128"; - put ""; - put ""; - put ''; - put ""; - length string $10000; - string=cats(''); - put string; - put ""; - put ""; - run; - proc metadata in=inref out=response; - run; -%end; - -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put "/GetMetadataObjects/Objects/Person"; - put ""; - put ''; - put "/GetMetadataObjects/Objects/Person/@Id"; - put "characterstring32"; - put ''; - put "/GetMetadataObjects/Objects/Person/@Name"; - put "characterstring256"; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -proc sort data= _XML_.SASObjects out=&outds; - by name; -run; - -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_getusers; diff --git a/003_macros/mm_getwebappsrvprops.sas b/003_macros/mm_getwebappsrvprops.sas deleted file mode 100644 index ab1e584..0000000 --- a/003_macros/mm_getwebappsrvprops.sas +++ /dev/null @@ -1,135 +0,0 @@ -/*** HELP START ***//** - @file - @brief Retrieves properties of the SAS web app server - @details - Usage: - - %mm_getwebappsrvprops(outds= some_ds) - data _null_; - set some_ds(where=(name='webappsrv.server.url')); - put value=; - run; - - @param [out] outds= the dataset to create that contains the list of properties - - @returns outds dataset containing all properties - - @warning The following filenames are created and then de-assigned: - - filename __in clear; - filename __out clear; - libname __shake clear; - - @version 9.4 - @author Allan Bowe https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mm_getwebappsrvprops( - outds= mm_getwebappsrvprops -)/*/STORE SOURCE*/; - -filename __in temp lrecl=10000; -filename __out temp lrecl=10000; -filename __shake temp lrecl=10000; -data _null_ ; - file __in ; - put '' ; - put '$METAREPOSITORY' ; - put 'TextStore' ; - put 'SAS' ; - put '388' ; - put '' ; - put ''; - put '' ; - put '' ; - put '' ; - put '' ; - put '' ; - put '' ; -run ; -proc metadata in=__in out=__out verbose;run; - -/* find the beginning of the text */ -%local start; -%let start=0; -data _null_; - infile __out lrecl=10000; - input; - length cleartemplate $32000; - cleartemplate=tranwrd(_infile_,'StoredText=""',''); - start=index(cleartemplate,'StoredText="'); - if start then do; - call symputx("start",start+11+length('StoredText=""')-1); - putlog cleartemplate ; - end; - stop; -run; -%put &=start; -%if &start>0 %then %do; - /* read the content, byte by byte, resolving escaped chars */ - data _null_; - length filein 8 fileid 8; - filein = fopen("__out","I",1,"B"); - fileid = fopen("__shake","O",1,"B"); - rec = "20"x; - length entity $6; - do while(fread(filein)=0); - x+1; - if x>&start then do; - rc = fget(filein,rec,1); - if rec='"' then leave; - else if rec="&" then do; - entity=rec; - do until (rec=";"); - if fread(filein) ne 0 then goto getout; - rc = fget(filein,rec,1); - entity=cats(entity,rec); - end; - select (entity); - when ('&' ) rec='&' ; - when ('<' ) rec='<' ; - when ('>' ) rec='>' ; - when (''') rec="'" ; - when ('"') rec='"' ; - when (' ') rec='0A'x; - when (' ') rec='0D'x; - when ('$' ) rec='$' ; - when (' ') rec='09'x; - otherwise putlog "%str(WARN)ING: missing value for " entity=; - end; - rc =fput(fileid, substr(rec,1,1)); - rc =fwrite(fileid); - end; - else do; - rc =fput(fileid,rec); - rc =fwrite(fileid); - end; - end; - end; - getout: - rc=fclose(filein); - rc=fclose(fileid); - run; - data &outds ; - infile __shake dlm='=' missover; - length name $50 value $500; - input name $ value $; - run; -%end; -%else %do; - %put NOTE: Unable to retrieve Web App Server Properties; - data &outds; - length name $50 value $500; - run; -%end; - -/* clear references */ -filename __in clear; -filename __out clear; -filename __shake clear; - -%mend mm_getwebappsrvprops; diff --git a/003_macros/mm_spkexport.sas b/003_macros/mm_spkexport.sas deleted file mode 100644 index 2be9bf8..0000000 --- a/003_macros/mm_spkexport.sas +++ /dev/null @@ -1,146 +0,0 @@ -/*** HELP START ***//** - @file mm_spkexport.sas - @brief Creates an batch spk export command - @details Creates a script that will export everything in a metadata folder to - a specified location. - If you have XCMD enabled, then you can use mmx_spkexport (which performs - the actual export) - - Note - the batch tools require a username and password. For security, - these are expected to have been provided in a protected directory. - - Usage: - - %* import the macros (or make them available some other way); - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* create sample text file as input to the macro; - filename tmp temp; - data _null_; - file tmp; - put '%let mmxuser="sasdemo";'; - put '%let mmxpass="Mars321";'; - run; - - filename myref "%sysfunc(pathname(work))/mmxexport.sh" - permission='A::u::rwx,A::g::r-x,A::o::---'; - %mm_spkexport(metaloc=%str(/my/meta/loc) - ,outref=myref - ,secureref=tmp - ,cmdoutloc=%str(/tmp) - ) - - Alternatively, call without inputs to create a function style output - - filename myref "/tmp/mmscript.sh" - permission='A::u::rwx,A::g::r-x,A::o::---'; - %mm_spkexport(metaloc=%str(/my/meta/loc) - outref=myref - ,cmdoutloc=%str(/tmp) - ,cmdoutname=mmx - ) - - You can then navigate and execute as follows: - - cd /tmp - ./mmscript.sh "myuser" "mypass" - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mf_isblank.sas - @li mf_loc.sas - @li mm_tree.sas - @li mp_abort.sas - - - @param [in] metaloc= the metadata folder to export - @param [in] secureref= fileref containing the username / password (should - point to a file in a secure location). Leave blank to substitute $bash vars. - @param [in] excludevars= (0) A space seperated list of macro variable names, - each of which contains a value that should be used to filter the output - objects. - @param [out] outref= fileref to which to write the command - @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the - command will write the SPK - @param [out] cmdoutname= (mmxport) The name of the spk / log files to create - (will be identical just with .spk or .log extension) - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_spkexport(metaloc= - ,secureref= - ,excludevars=0 - ,outref= - ,cmdoutloc=%sysfunc(pathname(work)) - ,cmdoutname=mmxport -); - -%if &sysscp=WIN %then %do; - %put %str(WARN)ING: the script has been written assuming a unix system; - %put %str(WARN)ING- it will run anyway as should be easy to modify; -%end; - -/* set creds */ -%local mmxuser mmxpath i var; -%let mmxuser=$1; -%let mmxpass=$2; -%if %mf_isblank(&secureref)=0 %then %do; - %inc &secureref/nosource; -%end; - -/* setup metadata connection options */ -%local host port platform_object_path ds; -%let host=%sysfunc(getoption(metaserver)); -%let port=%sysfunc(getoption(metaport)); -%let platform_object_path=%mf_loc(POF); -%let ds=%mf_getuniquename(prefix=spkexportable); - -%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds) - -%if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref(); - -data _null_; - set &ds end=last; - file &outref lrecl=32767; - length str $32767; - if _n_=1 then do; - put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)"; - put "cd ""&platform_object_path"" \"; - put "; ./ExportPackage -host &host -port &port -user &mmxuser \"; - put " -disableX11 -password &mmxpass \"; - put " -package ""&cmdoutloc/&cmdoutname..spk"" \"; - end; -/* exclude particular patterns from the exported SPK */ -%if "&excludevars" ne "0" %then %do; - %do i=1 %to %sysfunc(countw(&excludevars)); - %let var=%scan(&excludevars,&i); - if _n_=1 then do; - length excludestr&i $1000; - retain excludestr&i; - excludestr&i=symget("&var"); - putlog excludestr&i=; - putlog path=; - end; - if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0; - %end; - /* ignore top level folder else all subcontent will be exported regardless */ - if _n_>1; -%end; - str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \'); - put str; - if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 "; -run; - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=mm_spkexport - ,msg=%str(syscc=&syscc) -) - -%mend mm_spkexport; diff --git a/003_macros/mm_tree.sas b/003_macros/mm_tree.sas deleted file mode 100644 index ca7efe9..0000000 --- a/003_macros/mm_tree.sas +++ /dev/null @@ -1,173 +0,0 @@ -/*** HELP START ***//** - @file mm_tree.sas - @brief Returns all folders / subfolder content for a particular root - @details Shows all members and SubTrees for a particular root. - - Model: - - metauri char(64), - name char(256) format=$256. informat=$256. label='name', - path char(1024), - publictype char(32), - MetadataUpdated char(32), - MetadataCreated char(32) - - Usage: - - %* load macros; - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* export everything; - %mm_tree(root= ,outds=iwantthisdataset) - - %* export everything in a specific folder; - %mm_tree(root=%str(/my/folder) ,outds=stuff) - - %* export only folders; - %mm_tree(root=%str(/my/folder) ,types=Folder ,outds=stuf) - - %* show only exportable content; - %mm_tree(root=%str(/) ,types=EXPORTABLE ,outds=exportable) - - %* with specific types; - %mm_tree(root=%str(/my/folder) - ,types= - DeployedJob - ExternalFile - Folder - Folder.SecuredData - GeneratedTransform - InformationMap.Relational - Job - Library - Prompt - StoredProcess - Table - ,outds=morestuff) - -

SAS Macros

- @li mf_getquotedstr.sas - @li mm_getpublictypes.sas - @li mf_isblank.sas - - @param [in] root= the parent folder under which to return all contents - @param [in] outds= the dataset to create that contains the list of directories - @param [in] types= Space-seperated, unquoted list of types for filtering the - output. Special types: - - * ALl - return all types (the default) - * EXPORTABLE - return only the content types that can be exported in an SPK - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ -%macro mm_tree( - root= - ,types=ALL - ,outds=work.mm_tree -)/*/STORE SOURCE*/; -options noquotelenmax; - -%if %mf_isblank(&root) %then %let root=/; - -%if %str(&types)=EXPORTABLE %then %do; - data;run;%local tempds; %let tempds=&syslast; - %mm_getpublictypes(outds=&tempds) - proc sql noprint; - select publictype into: types separated by ' ' from &tempds; - drop table &tempds; -%end; - -* use a temporary fileref to hold the response; -filename response temp; -/* get list of libraries */ -proc metadata in= - '$METAREPOSITORY - TreeSAS - 384 - - ' - out=response; -run; -/* -data _null_; - infile response; - input; - put _infile_; - run; -*/ - -/* create an XML map to read the response */ -filename sxlemap temp; -data _null_; - file sxlemap; - put ''; - put "/GetMetadataObjects/Objects/Tree"; - put ""; - put ''; - put "/GetMetadataObjects/Objects/Tree/@Id"; - put "characterstring64"; - put ''; - put "/GetMetadataObjects/Objects/Tree/@Name"; - put "characterstring256"; - put '
'; -run; -libname _XML_ xml xmlfileref=response xmlmap=sxlemap; - -data &outds; - length metauri pathuri $64 name $256 path $1024 - publictype MetadataUpdated MetadataCreated $32; - set _XML_.SASObjects; - keep metauri name publictype MetadataUpdated MetadataCreated path; - length parenturi pname $128 ; - call missing(parenturi,pname); - path=cats('/',name); - /* get parents */ - tmpuri=pathuri; - do while (metadata_getnasn(tmpuri,"ParentTree",1,parenturi)>0); - rc=metadata_getattr(parenturi,"Name",pname); - path=cats('/',pname,path); - tmpuri=parenturi; - end; - - if path=:"&root"; - - %if "&types"="ALL" or ("&types" ne "ALL" and "&types" ne "Folder") %then %do; - n=1; - do while (metadata_getnasn(pathuri,"Members",n,metauri)>0); - n+1; - call missing(name,publictype,MetadataUpdated,MetadataCreated); - rc=metadata_getattr(metauri,"Name", name); - rc=metadata_getattr(metauri,"MetadataUpdated", MetadataUpdated); - rc=metadata_getattr(metauri,"MetadataCreated", MetadataCreated); - rc=metadata_getattr(metauri,"PublicType", PublicType); - %if "&types" ne "ALL" %then %do; - if publictype in (%mf_getquotedstr(&types)) then output; - %end; - %else output; ; - end; - %end; - - rc=metadata_resolve(pathuri,pname,tmpuri); - metauri=cats('OMSOBJ:',pname,'\',pathuri); - rc=metadata_getattr(metauri,"Name", name); - rc=metadata_getattr(pathuri,"MetadataUpdated", MetadataUpdated); - rc=metadata_getattr(pathuri,"MetadataCreated", MetadataCreated); - rc=metadata_getattr(pathuri,"PublicType", PublicType); - path=substr(path,1,length(path)-length(name)-1); - if publictype ne '' then output; -run; - -proc sort; - by path; -run; - -/* clear references */ -filename sxlemap clear; -filename response clear; -libname _XML_ clear; - -%mend mm_tree; diff --git a/003_macros/mm_updateappextension.sas b/003_macros/mm_updateappextension.sas deleted file mode 100644 index 93a2392..0000000 --- a/003_macros/mm_updateappextension.sas +++ /dev/null @@ -1,134 +0,0 @@ -/*** HELP START ***//** - @file - @brief Add or update an extension to an application component - @details A SAS Application (SoftwareComponent) is a great place to store app - specific parameters. There are two main places those params can be stored: - 1) Configuration, and 2) Extensions. The second location will enable end - users to modify parameters even if they don't have the Configuration Manager - plugin in SMC. This macro can be used after creating an application with - the mm_createapplication.sas macro. If a parameter with the same name - exists, it is updated. If it does not, it is created. - - Usage: - - %mm_updateappextension(app=/my/metadata/path/myappname - ,paramname=My Param - ,paramvalue=My value - ,paramdesc=some description) - - - @param [in] app= the BIP Tree folder path plus Application Name - @param [in] paramname= Parameter name - @param [in] paramvalue= Parameter value - @param [in] paramdesc= Parameter description - - @param [in] frefin= change default inref if it clashes with an existing one - @param [out] frefout= change default outref if it clashes with an existing one - @param [in] mDebug= set to 1 to show debug messages in the log - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_updateappextension(app= - ,paramname= - ,paramvalue= - ,paramdesc=Created by mm_updateappextension - ,frefin=inmeta,frefout=outmeta - , mdebug=0); - - -/* first, check if app (and param) exists */ -%local appuri exturi; -%let appuri=stopifempty; -%let exturi=createifempty; - -data _null_; - format type uri tsuri value $200.; - call missing (of _all_); - paramname=symget('paramname'); - path="&app(Application)"; - /* first, find the STP ID */ - if metadata_pathobj("",path,"Application",type,uri)>0 then do; - /* we have an app in this location! */ - call symputx('appuri',uri,'l'); - cnt=1; - do while (metadata_getnasn(uri,"Extensions",cnt,tsuri)>0); - rc=metadata_getattr(tsuri,"Name",value); - put tsuri= value=; - if value=paramname then do; - putlog "&sysmacroname: found existing param - " tsuri; - rc=metadata_getattr(tsuri,"Id",value); - call symputx('exturi',value,'l'); - stop; - end; - cnt+1; - end; - end; - else put (_all_)(=); -run; - -%if &appuri=stopifempty %then %do; - %put %str(WARN)ING: &app.(Application) not found!; - %return; -%end; - -/* escape the description so it can be stored as XML */ -data _null_; - length outstr $32767; - outstr=symget('paramdesc'); - outstr=tranwrd(outstr,'&','&'); - outstr=tranwrd(outstr,'<','<'); - outstr=tranwrd(outstr,'>','>'); - outstr=tranwrd(outstr,"'",'''); - outstr=tranwrd(outstr,'"','"'); - outstr=tranwrd(outstr,'0A'x,' '); - outstr=tranwrd(outstr,'0D'x,' '); - outstr=tranwrd(outstr,'$','$'); - call symputx('paramdesc',outstr,'l'); -run; - -filename &frefin temp; - -%if &exturi=createifempty %then %do; - /* write header XML */ - data _null_; - file &frefin; - pname=quote(trim(symget('paramname'))); - pdesc=quote(trim(symget('paramdesc'))); - pvalue=quote(trim(symget('paramvalue'))); - put "$METAREPOSITORY"/ - " " / - ' ' / - ' '/ - 'SAS268435456'; - run; - -%end; -%else %do; - data _null_; - file &frefin; - pdesc=quote(trim(symget('paramdesc'))); - pvalue=quote(trim(symget('paramvalue'))); - put "$METAREPOSITORY"/ - " 0 then do; - /* get sourcetext */ - cnt=1; - do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0); - rc=metadata_getattr(tsuri,"Name",value); - put tsuri= value=; - if value="&name" then do; - /* found it! */ - rc=metadata_getattr(tsuri,"Id",value); - call symputx('tsuri',value,'l'); - stop; - end; - cnt+1; - end; - end; - else put (_all_)(=); -run; - -%if &tsuri=stopifempty %then %do; - %put %str(WARN)ING: &path/&name.(Document) not found!; - %return; -%end; - -%if %length(&text)<2 %then %do; - %put %str(WARN)ING: No text supplied!!; - %return; -%end; - -filename &frefin temp recfm=n; - -/* escape code so it can be stored as XML */ -/* input file may be over 32k wide, so deal with one char at a time */ -data _null_; - file &frefin recfm=n; - infile &text recfm=n; - input instr $CHAR1. ; - if _n_=1 then put "$METAREPOSITORY - ') put '>'; - when ("'") put '''; - when ('"') put '"'; - when ('0A'x) put ' '; - when ('0D'x) put ' '; - when ('$') put '$'; - otherwise put instr $CHAR1.; - end; -run; - -data _null_; - file &frefin mod; - put "'>SAS268435456 - "; -run; - - -filename &frefout temp; - -proc metadata in= &frefin - %if &mdebug=1 %then out=&frefout verbose; -; -run; - -%if &mdebug=1 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile &frefout lrecl=1048576; - input; - put _infile_; - run; -%end; - -%mend mm_updatedocument; diff --git a/003_macros/mm_updatestpservertype.sas b/003_macros/mm_updatestpservertype.sas deleted file mode 100644 index 6d37399..0000000 --- a/003_macros/mm_updatestpservertype.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file - @brief Updates a type 2 stored process to run on STP or WKS context - @details Only works on Type 2 (9.3 compatible) STPs - - Usage: - - %mm_updatestpservertype(target=/some/meta/path/myStoredProcess - ,type=WKS) - - - @param [in] target= full path to the STP being deleted - @param [in] type= Either WKS or STP depending on whether Workspace or - Stored Process type required - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mm_updatestpservertype( - target= - ,type= -)/*/STORE SOURCE*/; - -/** - * Check STP does exist - */ -%local cmtype; -data _null_; - length type uri $256; - rc=metadata_pathobj("","&target",'StoredProcess',type,uri); - call symputx('cmtype',type,'l'); - call symputx('stpuri',uri,'l'); -run; -%if &cmtype ne ClassifierMap %then %do; - %put %str(WARN)ING: No Stored Process found at ⌖ - %return; -%end; - -%local newtype; -%if &type=WKS %then %let newtype=Wks; -%else %let newtype=Sps; - -%local result; -%let result=NOT FOUND; -data _null_; - length uri name value $256; - n=1; - do while(metadata_getnasn("&stpuri","Notes",n,uri)>0); - n+1; - rc=metadata_getattr(uri,"Name",name); - if name='Stored Process' then do; - rc = METADATA_SETATTR(uri,'StoredText' - ,'' - !!''); - if rc=0 then call symputx('result','SUCCESS'); - stop; - end; - end; -run; -%if &result=SUCCESS %then %put NOTE: SUCCESS: STP &target changed to &type type; -%else %put %str(ERR)OR: Issue with &sysmacroname; - -%mend mm_updatestpservertype; diff --git a/003_macros/mm_updatestpsourcecode.sas b/003_macros/mm_updatestpsourcecode.sas deleted file mode 100644 index e6ec26c..0000000 --- a/003_macros/mm_updatestpsourcecode.sas +++ /dev/null @@ -1,133 +0,0 @@ -/*** HELP START ***//** - @file - @brief Update the source code of a type 2 STP - @details Uploads the contents of a text file or fileref to an existing type 2 - STP. A type 2 STP has its source code saved in metadata. - - Usage: - - %mm_updatestpsourcecode(stp=/my/metadata/path/mystpname - ,stpcode="/file/system/source.sas") - - @param [in] stp= the BIP Tree folder path plus Stored Process Name - @param [in] stpcode= () The source file (or fileref) containing the SAS - code to load into the stp. - For multiple files, they should simply be concatenated first. - @param [in] minify= (NO) Set to YES in order to strip comments, blank lines, - and CRLFs. - @param [in] mDebug= set to 1 to show debug messages in the log - - @version 9.3 - @author Allan Bowe - -

SAS Macros

- @li mf_getuniquefileref.sas - -**//*** HELP END ***/ - -%macro mm_updatestpsourcecode(stp= - ,stpcode= - ,minify=NO - ,mdebug=0 -); - -/* first, check if STP exists */ -%local tsuri; -%let tsuri=stopifempty ; - -data _null_; - format type uri tsuri value $200.; - call missing (of _all_); - path="&stp.(StoredProcess)"; - /* first, find the STP ID */ - if metadata_pathobj("",path,"StoredProcess",type,uri)>0 then do; - /* get sourcecode */ - cnt=1; - do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0); - rc=metadata_getattr(tsuri,"Name",value); - %if &mdebug=1 %then %do; - put tsuri= value=; - %end; - if value="SourceCode" then do; - /* found it! */ - rc=metadata_getattr(tsuri,"Id",value); - call symputx('tsuri',value,'l'); - stop; - end; - cnt+1; - end; - end; - else put (_all_)(=); -run; - -%if &tsuri=stopifempty %then %do; - %put %str(WARN)ING: &stp.(StoredProcess) not found!; - %return; -%end; - -%if %length(&stpcode)<2 %then %do; - %put %str(WARN)ING: No SAS code supplied!!; - %return; -%end; - -%local frefin frefout; -%let frefin=%mf_getuniquefileref(); -%let frefout=%mf_getuniquefileref(); - -/* write header XML */ -data _null_; - file &frefin; - put "$METAREPOSITORY - ','>'); - outstr=tranwrd(outstr,"'",'''); - outstr=tranwrd(outstr,'"','"'); - outstr=tranwrd(outstr,'0A'x,' '); - outstr=tranwrd(outstr,'0D'x,' '); - outstr=tranwrd(outstr,'$','$'); - %if &minify=YES %then %do; - outstr=cats(outstr); - if outstr ne ''; - if not (outstr=:'/*' and subpad(left(reverse(outstr)),1,2)='/*'); - %end; - outstr=trim(outstr); - put outstr ' '; - run; -%end; - -data _null_; - file &frefin mod; - put "'>SAS268435456 - "; -run; - -proc metadata in= &frefin out=&frefout; -run; - -%if &mdebug=1 %then %do; - /* write the response to the log for debugging */ - data _null_; - infile &frefout lrecl=32767; - input; - put _infile_; - run; -%end; -%else %do; - filename &frefin clear; - filename &frefout clear; -%end; - -%mend mm_updatestpsourcecode; diff --git a/003_macros/mm_webout.sas b/003_macros/mm_webout.sas deleted file mode 100644 index cb00533..0000000 --- a/003_macros/mm_webout.sas +++ /dev/null @@ -1,251 +0,0 @@ -/*** HELP START ***//** - @file mm_webout.sas - @brief Send data to/from SAS Stored Processes - @details This macro should be added to the start of each Stored Process, - **immediately** followed by a call to: - - %mm_webout(FETCH) - - This will read all the input data and create same-named SAS datasets in the - WORK library. You can then insert your code, and send data back using the - following syntax: - - data some datasets; * make some data ; - retain some columns; - run; - - %mm_webout(OPEN) - %mm_webout(ARR,some) * Array format, fast, suitable for large tables ; - %mm_webout(OBJ,datasets) * Object format, easier to work with ; - - Finally, wrap everything up send some helpful system variables too - - %mm_webout(CLOSE) - - - @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE - @param [in] ds The dataset to send back to the frontend - @param [out] dslabel= Value to use instead of table name for sending to JSON - @param [in] fmt= (N) Setting Y converts all vars to their formatted values - @param [out] fref= (_webout) The fileref to which to write the JSON - @param [in] missing= (NULL) Special numeric missing values can be sent as NULL - (eg `null`) or as STRING values (eg `".a"` or `".b"`) - @param [in] showmeta= (N) Set to Y to output metadata alongside each table, - such as the column formats and types. The metadata is contained inside an - object with the same name as the table but prefixed with a dollar sign - ie, - `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` - @param [in] workobs= (0) When set to a positive integer, will create a new - output object (WORK) which contains this number of observations from all - tables in the WORK library. - @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows - that should be converted to output JSON - -

SAS Macros

- @li mp_jsonout.sas - -

Related Macros

- @li ms_webout.sas - @li mv_webout.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ -%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL - ,showmeta=N,maxobs=MAX,workobs=0 -); -%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug - sasjs_tables; -%local i tempds jsonengine; - -/* see https://github.com/sasjs/core/issues/41 */ -%if "%upcase(&SYSENCODING)" ne "UTF-8" %then %let jsonengine=PROCJSON; -%else %let jsonengine=DATASTEP; - - -%if &action=FETCH %then %do; - %if %str(&_debug) ge 131 %then %do; - options mprint notes mprintnest; - %end; - %let _webin_file_count=%eval(&_webin_file_count+0); - /* now read in the data */ - %do i=1 %to &_webin_file_count; - %if &_webin_file_count=1 %then %do; - %let _webin_fileref1=&_webin_fileref; - %let _webin_name1=&_webin_name; - %end; - data _null_; - infile &&_webin_fileref&i termstr=crlf; - input; - call symputx('input_statement',_infile_); - putlog "&&_webin_name&i input statement: " _infile_; - stop; - data &&_webin_name&i; - infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8'; - input &input_statement; - %if %str(&_debug) ge 131 %then %do; - if _n_<20 then putlog _infile_; - %end; - run; - %let sasjs_tables=&sasjs_tables &&_webin_name&i; - %end; -%end; - -%else %if &action=OPEN %then %do; - /* fix encoding */ - OPTIONS NOBOMFILE; - - /** - * check xengine type to avoid the below err message: - * > Function is only valid for filerefs using the CACHE access method. - */ - data _null_; - set sashelp.vextfl(where=(fileref="_WEBOUT")); - if xengine='STREAM' then do; - rc=stpsrv_header('Content-type',"text/html; encoding=utf-8"); - end; - run; - - /* setup json */ - data _null_;file &fref encoding='utf-8'; - %if %str(&_debug) ge 131 %then %do; - put '>>weboutBEGIN<<'; - %end; - put '{"SYSDATE" : "' "&SYSDATE" '"'; - put ',"SYSTIME" : "' "&SYSTIME" '"'; - run; - -%end; - -%else %if &action=ARR or &action=OBJ %then %do; - %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref - ,engine=&jsonengine,missing=&missing,showmeta=&showmeta,maxobs=&maxobs - ) -%end; -%else %if &action=CLOSE %then %do; - /* To avoid issues with _webout on EBI we use a temporary file */ - filename _sjsref temp lrecl=131068; - %if %str(&workobs) > 0 %then %do; - /* if debug mode, send back first XX records of each work table also */ - data;run;%let tempds=%scan(&syslast,2,.); - ods output Members=&tempds; - proc datasets library=WORK memtype=data; - %local wtcnt;%let wtcnt=0; - data _null_; - set &tempds; - if not (upcase(name) =:"DATA"); /* ignore temp datasets */ - i+1; - call symputx(cats('wt',i),name,'l'); - call symputx('wtcnt',i,'l'); - data _null_; file _sjsref mod encoding='utf-8'; - put ",""WORK"":{"; - %do i=1 %to &wtcnt; - %let wt=&&wt&i; - data _null_; file _sjsref mod encoding='utf-8'; - dsid=open("WORK.&wt",'is'); - nlobs=attrn(dsid,'NLOBS'); - nvars=attrn(dsid,'NVARS'); - rc=close(dsid); - if &i>1 then put ','@; - put " ""&wt"" : {"; - put '"nlobs":' nlobs; - put ',"nvars":' nvars; - %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y - ,maxobs=&workobs - ) - data _null_; file _sjsref mod encoding='utf-8'; - put "}"; - %end; - data _null_; file _sjsref mod encoding='utf-8'; - put "}"; - run; - %end; - /* close off json */ - data _null_;file _sjsref mod encoding='utf-8'; - length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; - put ",""_DEBUG"" : ""&_debug"" "; - _METAUSER=quote(trim(symget('_METAUSER'))); - put ",""_METAUSER"": " _METAUSER; - _METAPERSON=quote(trim(symget('_METAPERSON'))); - put ',"_METAPERSON": ' _METAPERSON; - _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); - put ',"_PROGRAM" : ' _PROGRAM ; - autoexec=quote(urlencode(trim(getoption('autoexec')))); - put ',"AUTOEXEC" : ' autoexec; - put ",""MF_GETUSER"" : ""%mf_getuser()"" "; - put ",""SYSCC"" : ""&syscc"" "; - put ",""SYSENCODING"" : ""&sysencoding"" "; - syserrortext=cats(symget('syserrortext')); - if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syserrortext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syserrortext) - )))))))))))))!!'"'; - end; - else syserrortext=cats('"',syserrortext,'"'); - put ',"SYSERRORTEXT" : ' syserrortext; - put ",""SYSHOSTNAME"" : ""&syshostname"" "; - put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; - put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; - SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); - put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; - put ",""SYSJOBID"" : ""&sysjobid"" "; - put ",""SYSSCPL"" : ""&sysscpl"" "; - put ",""SYSSITE"" : ""&syssite"" "; - put ",""SYSUSERID"" : ""&sysuserid"" "; - sysvlong=quote(trim(symget('sysvlong'))); - put ',"SYSVLONG" : ' sysvlong; - syswarningtext=cats(symget('syswarningtext')); - if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syswarningtext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syswarningtext) - )))))))))))))!!'"'; - end; - else syswarningtext=cats('"',syswarningtext,'"'); - put ',"SYSWARNINGTEXT" : ' syswarningtext; - put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" '; - length memsize $32; - memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; - memsize=quote(cats(memsize)); - put ',"MEMSIZE" : ' memsize; - put "}" @; - %if %str(&_debug) ge 131 %then %do; - put '>>weboutEND<<'; - %end; - run; - /* now write to _webout 1 char at a time */ - data _null_; - infile _sjsref lrecl=1 recfm=n; - file &fref mod lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - filename _sjsref clear; - -%end; - -%mend mm_webout; diff --git a/003_macros/mv_createfile.sas b/003_macros/mv_createfile.sas deleted file mode 100644 index 5d52eb8..0000000 --- a/003_macros/mv_createfile.sas +++ /dev/null @@ -1,129 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a file in SAS Drive - @details Creates a file in SAS Drive and adds the appropriate content type. - If the parent folder does not exist, it is created. - - Usage: - - filename myfile temp; - data _null_; - file myfile; - put 'something'; - run; - %mv_createfile(path=/Public/temp,name=newfile.txt,inref=myfile) - - - @param [in] path= The parent folder in which to create the file - @param [in] name= The name of the file to be created - @param [in] inref= The fileref pointing to the file to be uploaded - @param [in] intype= (BINARY) The type of the input data. Valid values: - @li BINARY File is copied byte for byte using the mp_binarycopy.sas macro. - @li BASE64 File will be first decoded using the mp_base64.sas macro, then - loaded byte by byte to SAS Drive. - @param [in] contentdisp= (inline) Content Disposition. Example values: - @li inline - @li attachment - @param [in] ctype= (0) Set a default HTTP Content-Type header to be returned - with the file when the content is retrieved from the Files service. - @param [in] access_token_var= The global macro variable to contain the access - token, if using authorization_code grant type. - @param [in] grant_type= (sas_services) Valid values are: - @li password - @li authorization_code - @li sas_services - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - - @version VIYA V.03.05 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_isblank.sas - @li mp_abort.sas - @li mp_base64copy.sas - @li mp_binarycopy.sas - @li mv_createfolder.sas - -**//*** HELP END ***/ - -%macro mv_createfile(path= - ,name= - ,inref= - ,intype=BINARY - ,contentdisp=inline - ,ctype=0 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort(iftrue=(%mf_isblank(&path)=1 or %length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%mf_isblank(&name)=1 or %length(&name)=1) - ,mac=&sysmacroname - ,msg=%str(name value with length >1 must be provided) -) - -/* create folder if it does not already exist */ -%mv_createfolder(path=&path - ,access_token_var=&access_token_var - ,grant_type=&grant_type - ,mdebug=&mdebug -) - -/* create file with relevant options */ -%local fref; -%let fref=%mf_getuniquefileref(); -filename &fref filesrvc - folderPath="&path" - filename="&name" - cdisp="&contentdisp" -%if "&ctype" ne "0" %then %do; - ctype="&ctype" -%end; - lrecl=1048544; -%if &intype=BINARY %then %do; - %mp_binarycopy(inref=&inref, outref=&fref) -%end; -%else %if &intype=BASE64 %then %do; - %mp_base64copy(inref=&inref, outref=&fref, action=DECODE) -%end; - -filename &fref clear; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%put &sysmacroname: File &name successfully created in &path; -%put &sysmacroname:;%put; -%put &base_uri/SASJobExecution?_file=&path/&name;%put; -%put &sysmacroname:; - -%mend mv_createfile; diff --git a/003_macros/mv_createfolder.sas b/003_macros/mv_createfolder.sas deleted file mode 100644 index 7244534..0000000 --- a/003_macros/mv_createfolder.sas +++ /dev/null @@ -1,184 +0,0 @@ -/*** HELP START ***//** - @file mv_createfolder.sas - @brief Creates a viya folder if that folder does not already exist - @details Creates a viya folder by checking if each parent folder exists, and - recursively creating children if needed. - Usage: - - %mv_createfolder(path=/Public) - - - @param [in] path= The full path of the folder to be created - @param [in] access_token_var= The global macro variable to contain the access - token, if using authorization_code grant type. - @param [in] grant_type= (sas_services) Valid values are: - @li password - @li authorization_code - @li sas_services - - @param [in] mdebug=(0) set to 1 to enable DEBUG messages - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_isblank.sas - @li mf_getplatform.sas - @li mfv_existfolder.sas - - -**//*** HELP END ***/ - -%macro mv_createfolder(path= - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%if %mfv_existfolder(&path)=1 %then %do; - %put &sysmacroname: &path already exists; - %return; -%end; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) - -options noquotelenmax; - -%local subfolder_cnt; /* determine the number of subfolders */ -%let subfolder_cnt=%sysfunc(countw(&path,/)); - -%local href; /* resource address (none for root) */ -%let href="/folders/folders?parentFolderUri=/folders/folders/none"; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%local x newpath subfolder; -%do x=1 %to &subfolder_cnt; - %let subfolder=%scan(&path,&x,%str(/)); - %let newpath=&newpath/&subfolder; - - %local fname1; - %let fname1=%mf_getuniquefileref(); - - %put &sysmacroname checking to see if &newpath exists; - proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&newpath"; - %if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; - %end; - run; - %local libref1; - %let libref1=%mf_getuniquelibref(); - libname &libref1 JSON fileref=&fname1; - %mp_abort( - iftrue=( - &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 404 - ) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) - %if &mdebug=1 %then %do; - %put &sysmacroname following check to see if &newpath exists:; - %put _local_; - data _null_; - set &fname1; - input; - putlog _infile_; - run; - %end; - %if &SYS_PROCHTTP_STATUS_CODE=200 %then %do; - %*put &sysmacroname &newpath exists so grab the follow on link ; - data _null_; - set &libref1..links; - if rel='createChild' then - call symputx('href',quote(cats("&base_uri",href)),'l'); - run; - %end; - %else %if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put &sysmacroname &newpath not found - creating it now; - %local fname2; - %let fname2=%mf_getuniquefileref(); - data _null_; - length json $1000; - json=cats("'" - ,'{"name":' - ,quote(trim(symget('subfolder'))) - ,',"description":' - ,quote("&subfolder, created by &sysmacroname") - ,',"type":"folder"}' - ,"'" - ); - call symputx('json',json,'l'); - run; - - proc http method='POST' - in=&json - out=&fname2 - &oauth_bearer - url=%unquote(%superq(href)); - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - 'Content-Type'='application/vnd.sas.content.folder+json' - 'Accept'='application/vnd.sas.content.folder+json'; - run; - %put &=SYS_PROCHTTP_STATUS_CODE; - %put &=SYS_PROCHTTP_STATUS_PHRASE; - %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) - %local libref2; - %let libref2=%mf_getuniquelibref(); - libname &libref2 JSON fileref=&fname2; - %put &sysmacroname &newpath now created. Grabbing the follow on link ; - data _null_; - set &libref2..links; - if rel='createChild' then do; - call symputx('href',quote(cats("&base_uri",href)),'l'); - &dbg put (_all_)(=); - end; - run; - - libname &libref2 clear; - filename &fname2 clear; - %end; - filename &fname1 clear; - libname &libref1 clear; -%end; -%mend mv_createfolder; diff --git a/003_macros/mv_createjob.sas b/003_macros/mv_createjob.sas deleted file mode 100644 index 503acde..0000000 --- a/003_macros/mv_createjob.sas +++ /dev/null @@ -1,355 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a Viya Job - @details - Code is passed in as one or more filerefs. - - %* Step 1 - compile macros ; - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* Step 2 - Create some SAS code and add it to a job; - filename ft15f001 temp; - parmcards4; - data some_code; - set sashelp.class; - run; - ;;;; - %mv_createjob(path=/Public/app/sasjstemp/jobs/myjobs,name=myjob) - - The path to the job will then be shown in the log, eg as follows: - - ![viya job location](https://i.imgur.com/XRUDHgA.png) - - -

SAS Macros

- @li mp_abort.sas - @li mv_createfolder.sas - @li mf_getuniquelibref.sas - @li mf_getuniquefileref.sas - @li mf_getplatform.sas - @li mf_isblank.sas - @li mv_deletejes.sas - - @param [in] path= The full path (on SAS Drive) where the job will be created - @param [in] name= The name of the job - @param [in] desc= (Created by the mv_createjob.sas macro) The job description - @param [in] precode= () - Space separated list of filerefs, pointing to the code that - needs to be attached to the beginning of the job - @param [in] code= (ft15f001) Fileref(s) of the actual code to be added - @param [in] access_token_var= (ACCESS_TOKEN) - Global macro variable containing the access token - @param [in] grant_type= (sas_services) Valid values: - @li sas_services - @li detect - @li authorization_code - @li password - @param [in] replace= (YES) select NO to avoid replacing any existing job - @param [in] addjesbeginendmacros= (false) - Relates to the `_addjesbeginendmacros` setting. - Normally this would always be false however due to a Viya bug - (https://github.com/sasjs/cli/issues/1229) this is now configurable. Valid - values: - @li true - @li false - @li 0 - this will prevent the flag from being set (job will default to true) - @param [in] contextname= () Choose a specific context on which to run the Job. - Leave blank to use the default context. - From Viya 3.5 it is possible to configure a shared context - see -https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en - - @version VIYA V.03.04 - @author [Allan Bowe](https://www.linkedin.com/in/allanbowe) - -**//*** HELP END ***/ - -%macro mv_createjob(path= - ,name= - ,desc=Created by the mv_createjob.sas macro - ,precode= - ,code=ft15f001 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,replace=YES - ,debug=0 - ,contextname= - ,addjesbeginendmacros=false - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -/* initial validation checking */ -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%mf_isblank(&name)=1) - ,mac=&sysmacroname - ,msg=%str(name value must be provided) -) - -options noquotelenmax; - -* remove any trailing slash ; -%if "%substr(&path,%length(&path),1)" = "/" %then - %let path=%substr(&path,1,%length(&path)-1); - -/* ensure folder exists */ -%put &sysmacroname: Path &path being checked / created; -%mv_createfolder(path=&path) - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&path"; -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%if &debug %then %do; - data _null_; - infile &fname1; - input; - putlog _infile_; - run; -%end; -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -/* path exists. Grab follow on link to check members */ -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; - -data _null_; - set &libref1..links; - if rel='members' then call symputx('membercheck',quote("&base_uri"!!trim(href)),'l'); - else if rel='self' then call symputx('parentFolderUri',href,'l'); -run; -data _null_; - set &libref1..root; - call symputx('folderid',id,'l'); -run; -%local fname2; -%let fname2=%mf_getuniquefileref(); -proc http method='GET' - out=&fname2 - &oauth_bearer - url=%unquote(%superq(membercheck)); - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - 'Accept'='application/vnd.sas.collection+json' - 'Accept-Language'='string'; -%if &debug=1 %then %do; - debug level = 3; -%end; -run; -/*data _null_;infile &fname2;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -%if %upcase(&replace)=YES %then %do; - %mv_deletejes(path=&path, name=&name) -%end; -%else %do; - /* check that job does not already exist in that folder */ - %local libref2; - %let libref2=%mf_getuniquelibref(); - libname &libref2 JSON fileref=&fname2; - %local exists; %let exists=0; - data _null_; - set &libref2..items; - if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then - call symputx('exists',1,'l'); - run; - %mp_abort(iftrue=(&exists=1) - ,mac=&sysmacroname - ,msg=%str(Job &name already exists in &path) - ) - libname &libref2 clear; -%end; - -/* set up the body of the request to create the service */ -%local fname3 comma; -%let fname3=%mf_getuniquefileref(); -data _null_; - file &fname3 TERMSTR=' '; - length string $32767; - string=cats('{"version": 0,"name":"' - ,"&name" - ,'","type":"Compute","parameters":[' -%if &addjesbeginendmacros ne 0 %then %do; - ,'{"name":"_addjesbeginendmacros"' - ,',"type":"CHARACTER","defaultValue":"' - ,"&addjesbeginendmacros" - ,'"}' - %let comma=%str(,); -%end; - ); - context=quote(cats(symget('contextname'))); - if context ne '""' then do; - string=cats(string - ,"&comma" - ,'{"version": 1,"name": "_contextName","defaultValue":' - ,context,',"type":"CHARACTER","label":"Context Name","required": false}' - ); - end; - string=cats(string,'],"code":"'); - put string; -run; - - -/* insert the code, escaping double quotes and carriage returns */ -%local x fref freflist; -%let freflist= &precode &code ; -%do x=1 %to %sysfunc(countw(&freflist)); - %let fref=%scan(&freflist,&x); - %put &sysmacroname: adding &fref; - data _null_; - length filein 8 fileid 8; - filein = fopen("&fref","I",1,"B"); - fileid = fopen("&fname3","A",1,"B"); - rec = "20"x; - do while(fread(filein)=0); - rc = fget(filein,rec,1); - if rec='"' then do; /* DOUBLE QUOTE */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'"');rc =fwrite(fileid); - end; - else if rec='0A'x then do; /* LF */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'n');rc =fwrite(fileid); - end; - else if rec='0D'x then do; /* CR */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'r');rc =fwrite(fileid); - end; - else if rec='09'x then do; /* TAB */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'t');rc =fwrite(fileid); - end; - else if rec='5C'x then do; /* BACKSLASH */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'\');rc =fwrite(fileid); - end; - else if rec='01'x then do; /* Unprintable */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'1');rc =fwrite(fileid); - end; - else if rec='07'x then do; /* Bell Char */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'7');rc =fwrite(fileid); - end; - else if rec='1B'x then do; /* escape char */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'1');rc =fwrite(fileid); - rc =fput(fileid,'B');rc =fwrite(fileid); - end; - else do; - rc =fput(fileid,rec); - rc =fwrite(fileid); - end; - end; - rc=fclose(filein); - rc=fclose(fileid); - run; -%end; - -/* finish off the body of the code file loaded to JES */ -data _null_; - file &fname3 mod TERMSTR=' '; - put '"}'; -run; - -/* now we can create the job!! */ -%local fname4; -%let fname4=%mf_getuniquefileref(); -proc http method='POST' - in=&fname3 - out=&fname4 - &oauth_bearer - url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri"; - headers 'Content-Type'='application/vnd.sas.job.definition+json' - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - "Accept"="application/vnd.sas.job.definition+json"; -%if &debug=1 %then %do; - debug level = 3; -%end; -run; -/*data _null_;infile &fname4;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) -/* clear refs */ -filename &fname1 clear; -filename &fname2 clear; -filename &fname3 clear; -filename &fname4 clear; -libname &libref1 clear; - -/* get the url so we can give a helpful log message */ -%local url; -data _null_; - if symexist('_baseurl') then do; - url=symget('_baseurl'); - if subpad(url,length(url)-9,9)='SASStudio' - then url=substr(url,1,length(url)-11); - else url="&systcpiphostname"; - end; - else url="&systcpiphostname"; - call symputx('url',url); -run; - - -%put &sysmacroname: Job &name successfully created in &path; -%put &sysmacroname:; -%put &sysmacroname: Check it out here:; -%put &sysmacroname:;%put; -%put &url/SASJobExecution?_PROGRAM=&path/&name;%put; -%put &sysmacroname:; -%put &sysmacroname:; - -%mend mv_createjob; diff --git a/003_macros/mv_createwebservice.sas b/003_macros/mv_createwebservice.sas deleted file mode 100644 index 1eaf54b..0000000 --- a/003_macros/mv_createwebservice.sas +++ /dev/null @@ -1,966 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a JobExecution web service if it doesn't already exist - @details - Code is passed in as one or more filerefs. - - %* Step 1 - compile macros ; - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* Step 2 - Create some code and add it to a web service; - filename ft15f001 temp; - parmcards4; - %webout(FETCH) %* fetch any tables sent from frontend; - %* do some sas, any inputs are now already WORK tables; - data example1 example2; - set sashelp.class; - run; - %* send data back; - %webout(OPEN) - %webout(ARR,example1) * Array format, fast, suitable for large tables; - %webout(OBJ,example2) * Object format, easier to work with ; - %webout(CLOSE) - ;;;; - %mv_createwebservice(path=/Public/app/common,name=appinit) - - - Notes: - To minimise postgres requests, output json is stored in a temporary file - and then sent to _webout in one go at the end. - -

SAS Macros

- @li mp_abort.sas - @li mv_createfolder.sas - @li mf_getuniquelibref.sas - @li mf_getuniquefileref.sas - @li mf_getplatform.sas - @li mf_isblank.sas - @li mv_deletejes.sas - - @param [in] path= The full path (on SAS Drive) where the service will be - created - @param [in] name= The name of the service - @param [in] desc= The description of the service - @param [in] precode= Space separated list of filerefs, pointing to the code - that needs to be attached to the beginning of the service - @param [in] code= Fileref(s) of the actual code to be added - @param [in] access_token_var= The global macro variable to contain the access - token - @param [in] grant_type= valid values are "password" or "authorization_code" - (unquoted). The default is authorization_code. - @param [in] replace=(YES) Select NO to avoid replacing any existing service in - that location - @param [in] adapter= the macro uses the sasjs adapter by default. To use - another adapter, add a (different) fileref here. - @param [in] contextname= Choose a specific context on which to run the Job. - Leave blank to use the default context. From Viya 3.5 it is possible to - configure a shared context - see -https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en - @param [in] mdebug=(0) set to 1 to enable DEBUG messages - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mv_createwebservice(path= - ,name= - ,desc=Created by the mv_createwebservice.sas macro - ,precode= - ,code=ft15f001 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,replace=YES - ,adapter=sasjs - ,mdebug=0 - ,contextname= - ,debug=0 /* @TODO - Deprecate */ - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -/* initial validation checking */ -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%mf_isblank(&name)=1) - ,mac=&sysmacroname - ,msg=%str(name value must be provided) -) - -options noquotelenmax; - -* remove any trailing slash ; -%if "%substr(&path,%length(&path),1)" = "/" %then - %let path=%substr(&path,1,%length(&path)-1); - -/* ensure folder exists */ -%put &sysmacroname: Path &path being checked / created; -%mv_createfolder(path=&path) - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&path"; -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%if &mdebug=1 %then %do; - data _null_; - infile &fname1; - input; - putlog _infile_; - run; -%end; -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -/* path exists. Grab follow on link to check members */ -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; - -data _null_; - set &libref1..links; - if rel='members' then - call symputx('membercheck',quote("&base_uri"!!trim(href)),'l'); - else if rel='self' then call symputx('parentFolderUri',href,'l'); -run; -data _null_; - set &libref1..root; - call symputx('folderid',id,'l'); -run; -%local fname2; -%let fname2=%mf_getuniquefileref(); -proc http method='GET' - out=&fname2 - &oauth_bearer - url=%unquote(%superq(membercheck)); - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - 'Accept'='application/vnd.sas.collection+json' - 'Accept-Language'='string'; -%if &mdebug=1 %then %do; - debug level = 3; -%end; -run; -/*data _null_;infile &fname2;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -%if %upcase(&replace)=YES %then %do; - %mv_deletejes(path=&path, name=&name) -%end; -%else %do; - /* check that job does not already exist in that folder */ - %local libref2; - %let libref2=%mf_getuniquelibref(); - libname &libref2 JSON fileref=&fname2; - %local exists; %let exists=0; - data _null_; - set &libref2..items; - if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then - call symputx('exists',1,'l'); - run; - %mp_abort(iftrue=(&exists=1) - ,mac=&sysmacroname - ,msg=%str(Job &name already exists in &path) - ) - libname &libref2 clear; -%end; - -/* set up the body of the request to create the service */ -%local fname3; -%let fname3=%mf_getuniquefileref(); -data _null_; - file &fname3 TERMSTR=' '; - length string $32767; - string=cats('{"version": 0,"name":"' - ,"&name" - ,'","type":"Compute","parameters":[{"name":"_addjesbeginendmacros"' - ,',"type":"CHARACTER","defaultValue":"false"}'); - context=quote(cats(symget('contextname'))); - if context ne '""' then do; - string=cats(string,',{"version": 1,"name": "_contextName","defaultValue":' - ,context,',"type":"CHARACTER","label":"Context Name","required": false}'); - end; - string=cats(string,'],"code":"'); - put string; -run; - -/** - * Add webout macro - * These put statements are auto generated - to change the macro, change the - * source (mv_webout) and run `build.py` - */ -filename &adapter temp lrecl=3000; -data _null_; - file &adapter; - put "/* Created on %sysfunc(datetime(),datetime19.) by &sysuserid */"; -/* WEBOUT BEGIN */ - put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y '; - put ' ,engine=DATASTEP '; - put ' ,missing=NULL '; - put ' ,showmeta=N '; - put ' ,maxobs=MAX '; - put ')/*/STORE SOURCE*/; '; - put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval '; - put ' tmpds1 tmpds2 tmpds3 tmpds4; '; - put '%let numcols=0; '; - put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); '; - put ' '; - put '%if &action=OPEN %then %do; '; - put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; - put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; - put ' run; '; - put '%end; '; - put '%else %if (&action=ARR or &action=OBJ) %then %do; '; - put ' /* force variable names to always be uppercase in the JSON */ '; - put ' options validvarname=upcase; '; - put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; - put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; - put ' filename _sjs1 temp lrecl=200 ; '; - put ' data _null_; file _sjs1 encoding=''utf-8''; '; - put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; - put ' run; '; - put ' /* now write to _webout 1 char at a time */ '; - put ' data _null_; '; - put ' infile _sjs1 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs1 clear; '; - put ' '; - put ' /* grab col defs */ '; - put ' proc contents noprint data=&ds '; - put ' out=_data_(keep=name type length format formatl formatd varnum label); '; - put ' run; '; - put ' %let colinfo=%scan(&syslast,2,.); '; - put ' proc sort data=&colinfo; '; - put ' by varnum; '; - put ' run; '; - put ' /* move meta to mac vars */ '; - put ' data &colinfo; '; - put ' if _n_=1 then call symputx(''numcols'',nobs,''l''); '; - put ' set &colinfo end=last nobs=nobs; '; - put ' name=upcase(name); '; - put ' /* fix formats */ '; - put ' if type=2 or type=6 then do; '; - put ' typelong=''char''; '; - put ' length fmt $49.; '; - put ' if format='''' then fmt=cats(''$'',length,''.''); '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else fmt=cats(format,formatl,''.''); '; - put ' end; '; - put ' else do; '; - put ' typelong=''num''; '; - put ' if format='''' then fmt=''best.''; '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else if formatd=0 then fmt=cats(format,formatl,''.''); '; - put ' else fmt=cats(format,formatl,''.'',formatd); '; - put ' end; '; - put ' /* 32 char unique name */ '; - put ' newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); '; - put ' '; - put ' call symputx(cats(''name'',_n_),name,''l''); '; - put ' call symputx(cats(''newname'',_n_),newname,''l''); '; - put ' call symputx(cats(''length'',_n_),length,''l''); '; - put ' call symputx(cats(''fmt'',_n_),fmt,''l''); '; - put ' call symputx(cats(''type'',_n_),type,''l''); '; - put ' call symputx(cats(''typelong'',_n_),typelong,''l''); '; - put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; - put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; - put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; - put ' run; '; - put ' '; - put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql; '; - put ' select count(*) into: lastobs from &ds; '; - put ' %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); '; - put ' '; - put ' %if &engine=PROCJSON %then %do; '; - put ' %if &missing=STRING %then %do; '; - put ' %put &sysmacroname: Special Missings not supported in proc json.; '; - put ' %put &sysmacroname: Switching to DATASTEP engine; '; - put ' %goto datastep; '; - put ' %end; '; - put ' data &tempds; '; - put ' set &ds; '; - put ' &stmt_obs; '; - put ' %if &fmt=N %then format _numeric_ best32.;; '; - put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; - put ' proc json out=_sjs2 pretty '; - put ' %if &action=ARR %then nokeys ; '; - put ' ;export &tempds / nosastags fmtnumeric; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs2 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs2 clear; '; - put ' %end; '; - put ' %else %if &engine=DATASTEP %then %do; '; - put ' %datastep: '; - put ' %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 '; - put ' %then %do; '; - put ' %put &sysmacroname: &ds NOT FOUND!!!; '; - put ' %return; '; - put ' %end; '; - put ' '; - put ' %if &fmt=Y %then %do; '; - put ' /** '; - put ' * Extract format definitions '; - put ' * First, by getting library locations from dictionary.formats '; - put ' * Then, by exporting the width using proc format '; - put ' * Cannot use maxw from sashelp.vformat as not always populated '; - put ' * Cannot use fmtinfo() as not supported in all flavours '; - put ' */ '; - put ' %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql noprint; '; - put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as FMTCAT, '; - put ' FMTNAME '; - put ' from dictionary.formats '; - put ' where fmttype=''F'' and libname is not null '; - put ' and fmtname in (select format from &colinfo where format is not null) '; - put ' order by 1; '; - put ' create table &tmpds2( '; - put ' FMTNAME char(32), '; - put ' LENGTH num '; - put ' ); '; - put ' %local catlist cat fmtlist i; '; - put ' select distinct fmtcat into: catlist separated by '' '' from &tmpds1; '; - put ' %do i=1 %to %sysfunc(countw(&catlist,%str( ))); '; - put ' %let cat=%scan(&catlist,&i,%str( )); '; - put ' proc sql; '; - put ' select distinct fmtname into: fmtlist separated by '' '' '; - put ' from &tmpds1 where fmtcat="&cat"; '; - put ' proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); '; - put ' select &fmtlist; '; - put ' run; '; - put ' proc sql; '; - put ' insert into &tmpds2 select distinct fmtname,length from &tmpds3; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' create table &tmpds4 as '; - put ' select a.*, b.length as MAXW '; - put ' from &colinfo a '; - put ' left join &tmpds2 b '; - put ' on cats(a.format)=cats(upcase(b.fmtname)) '; - put ' order by a.varnum; '; - put ' data _null_; '; - put ' set &tmpds4; '; - put ' if not missing(maxw); '; - put ' call symputx( '; - put ' cats(''fmtlen'',_n_), '; - put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; - put ' ,''l'' '; - put ' ); '; - put ' run; '; - put ' '; - put ' /* configure varlenchk - as we are explicitly shortening the variables */ '; - put ' %let optval=%sysfunc(getoption(varlenchk)); '; - put ' options varlenchk=NOWARN; '; - put ' data _data_(compress=char); '; - put ' /* shorten the new vars */ '; - put ' length '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i $&&fmtlen&i '; - put ' %end; '; - put ' ; '; - put ' /* rename on entry */ '; - put ' set &ds(rename=( '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i=&&newname&i '; - put ' %end; '; - put ' )); '; - put ' &stmt_obs; '; - put ' '; - put ' drop '; - put ' %do i=1 %to &numcols; '; - put ' &&newname&i '; - put ' %end; '; - put ' ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=num %then %do; '; - put ' &&name&i=cats(put(&&newname&i,&&fmt&i)); '; - put ' %end; '; - put ' %else %do; '; - put ' &&name&i=put(&&newname&i,&&fmt&i); '; - put ' %end; '; - put ' %end; '; - put ' if _error_ then do; '; - put ' call symputx(''syscc'',1012); '; - put ' stop; '; - put ' end; '; - put ' run; '; - put ' %let fmtds=&syslast; '; - put ' options varlenchk=&optval; '; - put ' %end; '; - put ' '; - put ' proc format; /* credit yabwon for special null removal */ '; - put ' value bart (default=40) '; - put ' %if &missing=NULL %then %do; '; - put ' ._ - .z = null '; - put ' %end; '; - put ' %else %do; '; - put ' ._ = [quote()] '; - put ' . = null '; - put ' .a - .z = [quote()] '; - put ' %end; '; - put ' other = [best.]; '; - put ' '; - put ' data &tempds; '; - put ' attrib _all_ label=''''; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' length &&name&i $&&fmtlen&i...; '; - put ' format &&name&i $&&fmtlen&i...; '; - put ' %end; '; - put ' %end; '; - put ' %if &fmt=Y %then %do; '; - put ' set &fmtds; '; - put ' %end; '; - put ' %else %do; '; - put ' set &ds; '; - put ' %end; '; - put ' &stmt_obs; '; - put ' format _numeric_ bart.; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' &&name&i=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; - put ' %end; '; - put ' %end; '; - put ' run; '; - put ' '; - put ' filename _sjs3 temp lrecl=131068 ; '; - put ' data _null_; '; - put ' file _sjs3 encoding=''utf-8''; '; - put ' if _n_=1 then put "["; '; - put ' set &tempds; '; - put ' if _n_>1 then put "," @; put '; - put ' %if &action=ARR %then "[" ; %else "{" ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &i>1 %then "," ; '; - put ' %if &action=OBJ %then """&&name&i"":" ; '; - put ' "&&name&i"n /* name literal for reserved variable names */ '; - put ' %end; '; - put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file _sjs3 mod encoding=''utf-8''; '; - put ' put '']''; '; - put ' run; '; - put ' data _null_; '; - put ' infile _sjs3 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs3 clear; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' drop table &colinfo, &tempds; '; - put ' '; - put ' %if %substr(&showmeta,1,1)=Y %then %do; '; - put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; '; - put ' file _sjs4; '; - put ' length label $350; '; - put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; - put ' do i=1 to &numcols; '; - put ' name=quote(trim(symget(cats(''name'',i)))); '; - put ' format=quote(trim(symget(cats(''fmt'',i)))); '; - put ' label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); '; - put ' length=quote(trim(symget(cats(''length'',i)))); '; - put ' type=quote(trim(symget(cats(''typelong'',i)))); '; - put ' if i>1 then put "," @@; '; - put ' put name '':{"format":'' format '',"label":'' label '; - put ' '',"length":'' length '',"type":'' type ''}''; '; - put ' end; '; - put ' put ''}}''; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs4 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs4 clear; '; - put ' %end; '; - put '%end; '; - put ' '; - put '%else %if &action=CLOSE %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod ; '; - put ' put "}"; '; - put ' run; '; - put '%end; '; - put '%mend mp_jsonout; '; - put ' '; - put '%macro mf_getuser( '; - put ')/*/STORE SOURCE*/; '; - put ' %local user; '; - put ' '; - put ' %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; '; - put ' %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; '; - put ' %let user=&SYS_COMPUTE_SESSION_OWNER; '; - put ' %end; '; - put ' %else %if %symexist(_metaperson) %then %do; '; - put ' %if %length(&_metaperson)=0 %then %let user=&sysuserid; '; - put ' /* sometimes SAS will add @domain extension - remove for consistency */ '; - put ' /* but be sure to quote in case of usernames with commas */ '; - put ' %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); '; - put ' %end; '; - put ' %else %let user=&sysuserid; '; - put ' '; - put ' %quote(&user) '; - put ' '; - put '%mend mf_getuser; '; - put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL '; - put ' ,showmeta=N,maxobs=MAX,workobs=0 '; - put '); '; - put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name '; - put ' sasjs_tables SYS_JES_JOB_URI; '; - put '%if %index("&_debug",log) %then %let _debug=131; '; - put ' '; - put '%local i tempds table; '; - put '%let action=%upcase(&action); '; - put ' '; - put '%if &action=FETCH %then %do; '; - put ' %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 131 %then %do; '; - put ' options mprint notes mprintnest; '; - put ' %end; '; - put ' '; - put ' %if not %symexist(_webin_fileuri1) %then %do; '; - put ' %let _webin_file_count=%eval(&_webin_file_count+0); '; - put ' %let _webin_fileuri1=&_webin_fileuri; '; - put ' %let _webin_name1=&_webin_name; '; - put ' %end; '; - put ' '; - put ' /* if the sasjs_tables param is passed, we expect param based upload */ '; - put ' %if %length(&sasjs_tables.X)>1 %then %do; '; - put ' '; - put ' /* convert data from macro variables to datasets */ '; - put ' %do i=1 %to %sysfunc(countw(&sasjs_tables)); '; - put ' %let table=%scan(&sasjs_tables,&i,%str( )); '; - put ' %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; '; - put ' data _null_; '; - put ' file "%sysfunc(pathname(work))/&table..csv" recfm=n; '; - put ' retain nrflg 0; '; - put ' length line $32767; '; - put ' do i=1 to &&sasjs&i.data0; '; - put ' if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); '; - put ' else line=symget(cats("sasjs&i.data",i)); '; - put ' if i=1 and substr(line,1,7)=''%nrstr('' then do; '; - put ' nrflg=1; '; - put ' line=substr(line,8); '; - put ' end; '; - put ' if i=&&sasjs&i.data0 and nrflg=1 then do; '; - put ' line=substr(line,1,length(line)-1); '; - put ' end; '; - put ' put line +(-1) @; '; - put ' end; '; - put ' run; '; - put ' data _null_; '; - put ' infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; '; - put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; - put ' list; '; - put ' data work.&table; '; - put ' infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd '; - put ' termstr=crlf; '; - put ' input &input_statement; '; - put ' run; '; - put ' %end; '; - put ' %end; '; - put ' %else %do i=1 %to &_webin_file_count; '; - put ' /* read in any files that are sent */ '; - put ' /* this part needs refactoring for wide files */ '; - put ' filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999; '; - put ' data _null_; '; - put ' infile indata termstr=crlf lrecl=32767; '; - put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' if _n_<20 then putlog _infile_; '; - put ' else stop; '; - put ' %end; '; - put ' %else %do; '; - put ' stop; '; - put ' %end; '; - put ' run; '; - put ' data &&_webin_name&i; '; - put ' infile indata firstobs=2 dsd termstr=crlf ; '; - put ' input &input_statement; '; - put ' run; '; - put ' %let sasjs_tables=&sasjs_tables &&_webin_name&i; '; - put ' %end; '; - put '%end; '; - put '%else %if &action=OPEN %then %do; '; - put ' /* setup webout */ '; - put ' OPTIONS NOBOMFILE; '; - put ' %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; '; - put ' filename _webout temp lrecl=999999 mod; '; - put ' %end; '; - put ' %else %do; '; - put ' filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" '; - put ' name="_webout.json" lrecl=999999 mod; '; - put ' %end; '; - put ' '; - put ' /* setup temp ref */ '; - put ' %if %upcase(&fref) ne _WEBOUT %then %do; '; - put ' filename &fref temp lrecl=999999 permission=''A::u::rwx,A::g::rw-,A::o::---''; '; - put ' %end; '; - put ' '; - put ' /* setup json */ '; - put ' data _null_;file &fref; '; - put ' put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; '; - put ' put '',"SYSTIME" : "'' "&SYSTIME" ''"''; '; - put ' run; '; - put '%end; '; - put '%else %if &action=ARR or &action=OBJ %then %do; '; - put ' %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref '; - put ' ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs '; - put ' ) '; - put '%end; '; - put '%else %if &action=CLOSE %then %do; '; - put ' %if %str(&workobs) > 0 %then %do; '; - put ' /* send back first XX records of each work table for debugging */ '; - put ' data;run;%let tempds=%scan(&syslast,2,.); '; - put ' ods output Members=&tempds; '; - put ' proc datasets library=WORK memtype=data; '; - put ' %local wtcnt;%let wtcnt=0; '; - put ' data _null_; '; - put ' set &tempds; '; - put ' if not (upcase(name) =:"DATA"); /* ignore temp datasets */ '; - put ' i+1; '; - put ' call symputx(cats(''wt'',i),name,''l''); '; - put ' call symputx(''wtcnt'',i,''l''); '; - put ' data _null_; file &fref mod; put ",""WORK"":{"; '; - put ' %do i=1 %to &wtcnt; '; - put ' %let wt=&&wt&i; '; - put ' data _null_; file &fref mod; '; - put ' dsid=open("WORK.&wt",''is''); '; - put ' nlobs=attrn(dsid,''NLOBS''); '; - put ' nvars=attrn(dsid,''NVARS''); '; - put ' rc=close(dsid); '; - put ' if &i>1 then put '',''@; '; - put ' put " ""&wt"" : {"; '; - put ' put ''"nlobs":'' nlobs; '; - put ' put '',"nvars":'' nvars; '; - put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y '; - put ' ,maxobs=&workobs '; - put ' ) '; - put ' data _null_; file &fref mod;put "}"; '; - put ' %end; '; - put ' data _null_; file &fref mod;put "}";run; '; - put ' %end; '; - put ' '; - put ' /* close off json */ '; - put ' data _null_;file &fref mod; '; - put ' length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; '; - put ' put ",""_DEBUG"" : ""&_debug"" "; '; - put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; - put ' put '',"_PROGRAM" : '' _PROGRAM ; '; - put ' autoexec=quote(urlencode(trim(getoption(''autoexec'')))); '; - put ' put '',"AUTOEXEC" : '' autoexec; '; - put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; - put ' SYS_JES_JOB_URI=quote(trim(resolve(symget(''SYS_JES_JOB_URI'')))); '; - put ' put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; '; - put ' put ",""SYSJOBID"" : ""&sysjobid"" "; '; - put ' put ",""SYSCC"" : ""&syscc"" "; '; - put ' syserrortext=cats(symget(''syserrortext'')); '; - put ' if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syserrortext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syserrortext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syserrortext=cats(''"'',syserrortext,''"''); '; - put ' put '',"SYSERRORTEXT" : '' syserrortext; '; - put ' put ",""SYSHOSTNAME"" : ""&syshostname"" "; '; - put ' put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; '; - put ' put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; '; - put ' SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); '; - put ' put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; '; - put ' put ",""SYSJOBID"" : ""&sysjobid"" "; '; - put ' put ",""SYSSCPL"" : ""&sysscpl"" "; '; - put ' put ",""SYSSITE"" : ""&syssite"" "; '; - put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; - put ' sysvlong=quote(trim(symget(''sysvlong''))); '; - put ' put '',"SYSVLONG" : '' sysvlong; '; - put ' syswarningtext=cats(symget(''syswarningtext'')); '; - put ' if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syswarningtext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syswarningtext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syswarningtext=cats(''"'',syswarningtext,''"''); '; - put ' put '',"SYSWARNINGTEXT" : '' syswarningtext; '; - put ' put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; '; - put ' length memsize $32; '; - put ' memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; '; - put ' memsize=quote(cats(memsize)); '; - put ' put '',"MEMSIZE" : '' memsize; '; - put ' put "}"; '; - put ' '; - put ' %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; '; - put ' data _null_; rc=fcopy("&fref","_webout");run; '; - put ' %end; '; - put ' '; - put '%end; '; - put ' '; - put '%mend mv_webout; '; -/* WEBOUT END */ - put '/* if calling viya service with _job param, _program will conflict */'; - put '/* so it is provided by SASjs instead as __program */'; - put '%global __program _program;'; - put '%let _program=%sysfunc(coalescec(&__program,&_program));'; - put ' '; - put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO'; - put ' ,maxobs=MAX'; - put ');'; - put ' %mv_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing'; - put ' ,showmeta=&showmeta,maxobs=&maxobs'; - put ' )'; - put '%mend;'; -run; - -/* insert the code, escaping double quotes and carriage returns */ -%&dbg.put &sysmacroname: Creating final input file; -%local x fref freflist; -%let freflist= &adapter &precode &code ; -%do x=1 %to %sysfunc(countw(&freflist)); - %let fref=%scan(&freflist,&x); - %&dbg.put &sysmacroname: adding &fref fileref; - data _null_; - length filein 8 fileid 8; - filein = fopen("&fref","I",1,"B"); - fileid = fopen("&fname3","A",1,"B"); - rec = "20"x; - do while(fread(filein)=0); - rc = fget(filein,rec,1); - if rec='"' then do; /* DOUBLE QUOTE */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'"');rc =fwrite(fileid); - end; - else if rec='0A'x then do; /* LF */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'n');rc =fwrite(fileid); - end; - else if rec='0D'x then do; /* CR */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'r');rc =fwrite(fileid); - end; - else if rec='09'x then do; /* TAB */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'t');rc =fwrite(fileid); - end; - else if rec='5C'x then do; /* BACKSLASH */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'\');rc =fwrite(fileid); - end; - else if rec='01'x then do; /* Unprintable */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'1');rc =fwrite(fileid); - end; - else if rec='07'x then do; /* Bell Char */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'7');rc =fwrite(fileid); - end; - else if rec='1B'x then do; /* escape char */ - rc =fput(fileid,'\');rc =fwrite(fileid); - rc =fput(fileid,'u');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'0');rc =fwrite(fileid); - rc =fput(fileid,'1');rc =fwrite(fileid); - rc =fput(fileid,'B');rc =fwrite(fileid); - end; - else do; - rc =fput(fileid,rec); - rc =fwrite(fileid); - end; - end; - rc=fclose(filein); - rc=fclose(fileid); - run; -%end; - -/* finish off the body of the code file loaded to JES */ -data _null_; - file &fname3 mod TERMSTR=' '; - put '"}'; -run; - -%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; - %put &sysmacroname: input about to be POSTed; - data _null_;infile &fname3;input;putlog _infile_;run; -%end; - -%&dbg.put &sysmacroname: Creating the actual service!; -%local fname4; -%let fname4=%mf_getuniquefileref(); -proc http method='POST' - in=&fname3 - out=&fname4 - &oauth_bearer - url="&base_uri/jobDefinitions/definitions?parentFolderUri=&parentFolderUri"; - headers 'Content-Type'='application/vnd.sas.job.definition+json' - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - "Accept"="application/vnd.sas.job.definition+json"; -%if &mdebug=1 %then %do; - debug level = 3; -%end; -run; -%if &mdebug=1 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then %do; - %put &sysmacroname: output from POSTing job definition; - data _null_;infile &fname4;input;putlog _infile_;run; -%end; -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 201) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -/* get the url so we can give a helpful log message */ -%local url; -data _null_; - if symexist('_baseurl') then do; - url=symget('_baseurl'); - if subpad(url,length(url)-9,9)='SASStudio' - then url=substr(url,1,length(url)-11); - else url="&systcpiphostname"; - end; - else url="&systcpiphostname"; - call symputx('url',url); -run; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%else %do; - /* clear refs */ - filename &fname1 clear; - filename &fname2 clear; - filename &fname3 clear; - filename &fname4 clear; - filename &adapter clear; - libname &libref1 clear; -%end; - -%put &sysmacroname: Job &name successfully created in &path; -%put &sysmacroname:; -%put &sysmacroname: Check it out here:; -%put &sysmacroname:;%put; -%put &url/SASJobExecution?_PROGRAM=&path/&name;%put; -%put &sysmacroname:; -%put &sysmacroname:; - -%mend mv_createwebservice; diff --git a/003_macros/mv_deletefoldermember.sas b/003_macros/mv_deletefoldermember.sas deleted file mode 100644 index e01feb6..0000000 --- a/003_macros/mv_deletefoldermember.sas +++ /dev/null @@ -1,154 +0,0 @@ -/*** HELP START ***//** - @file mv_deletefoldermember.sas - @brief Deletes an item in a Viya folder - @details If not executed in Studio 5+ will expect oauth token in a global - macro variable (default ACCESS_TOKEN). - - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %mv_createwebservice(path=/Public/test, name=blah) - %mv_deletejes(path=/Public/test, name=blah) - - - @param [in] path= () - The full path of the folder containing the item to be deleted - @param [in] name= The name of the item to be deleted - @param [in] contenttype= The contenttype of the item, eg: file, jobDefinition - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) - valid values are "password" or "authorization_code" (unquoted). - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_isblank.sas - -**//*** HELP END ***/ - -%macro mv_deletefoldermember(path= - ,name= - ,contenttype= - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%mf_isblank(&name)=1) - ,mac=&sysmacroname - ,msg=%str(name value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) - -options noquotelenmax; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%put &sysmacroname: fetching details for &path ; -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&path"; -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put &sysmacroname: Folder &path NOT FOUND - nothing to delete!; - %return; -%end; -%else %if &SYS_PROCHTTP_STATUS_CODE ne 200 %then %do; - /*data _null_;infile &fname1;input;putlog _infile_;run;*/ - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%put &sysmacroname: grab the follow on link ; -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; -data _null_; - set &libref1..links; - if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l'); -run; - -/* get the children */ -%local fname1a; -%let fname1a=%mf_getuniquefileref(); -proc http method='GET' out=&fname1a &oauth_bearer - url=%unquote(%superq(mref)); -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%put &=SYS_PROCHTTP_STATUS_CODE; -%local libref1a; -%let libref1a=%mf_getuniquelibref(); -libname &libref1a JSON fileref=&fname1a; -%local uri found; -%let found=0; -%put Getting object uri from &libref1a..items; -data _null_; - length contenttype name $1000; - set &libref1a..items; - if contenttype="&contenttype" and upcase(name)="%upcase(&name)" then do; - call symputx('uri',uri,'l'); - call symputx('found',1,'l'); - end; -run; -%if &found=0 %then %do; - %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ; - %return; -%end; -proc http method="DELETE" url="&base_uri&uri" &oauth_bearer; - headers -%if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" -%end; - "Accept"="*/*";/**/ -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 204 %then %do; - data _null_; infile &fname2; input; putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; -%else %put &sysmacroname: &path/&name(&contenttype) successfully deleted; - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; -filename &fname1a clear; -libname &libref1a clear; - -%mend mv_deletefoldermember; diff --git a/003_macros/mv_deletejes.sas b/003_macros/mv_deletejes.sas deleted file mode 100644 index e21a917..0000000 --- a/003_macros/mv_deletejes.sas +++ /dev/null @@ -1,151 +0,0 @@ -/*** HELP START ***//** - @file - @brief Deletes a Viya Job, if it exists - @details If not executed in Studio 5+ will expect oauth token in a global - macro variable (default ACCESS_TOKEN). - - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %mv_createwebservice(path=/Public/test, name=blah) - %mv_deletejes(path=/Public/test, name=blah) - - - @param [in] path= () - The full path of the folder containing the Job Execution Service - @param [in] name= The name of the Job Execution Service to be deleted - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) - Valid values are "password" or "authorization_code" (unquoted). - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_isblank.sas - -**//*** HELP END ***/ - -%macro mv_deletejes(path= - ,name= - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%mf_isblank(&name)=1) - ,mac=&sysmacroname - ,msg=%str(name value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%put &sysmacroname: fetching details for &path ; -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&path"; -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put &sysmacroname: Folder &path NOT FOUND - nothing to delete!; - %return; -%end; -%else %if &SYS_PROCHTTP_STATUS_CODE ne 200 %then %do; - /*data _null_;infile &fname1;input;putlog _infile_;run;*/ - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%put &sysmacroname: grab the follow on link ; -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; -data _null_; - set &libref1..links; - if rel='members' then call symputx('mref',quote("&base_uri"!!trim(href)),'l'); -run; - -/* get the children */ -%local fname1a; -%let fname1a=%mf_getuniquefileref(); -proc http method='GET' out=&fname1a &oauth_bearer - url=%unquote(%superq(mref)); -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%put &=SYS_PROCHTTP_STATUS_CODE; -%local libref1a; -%let libref1a=%mf_getuniquelibref(); -libname &libref1a JSON fileref=&fname1a; -%local uri found; -%let found=0; -%put Getting object uri from &libref1a..items; -data _null_; - length contenttype name $1000; - set &libref1a..items; - if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do; - call symputx('uri',cats("&base_uri",uri),'l'); - call symputx('found',1,'l'); - end; -run; -%if &found=0 %then %do; - %put NOTE:;%put NOTE- &sysmacroname: &path/&name NOT FOUND;%put NOTE- ; - %return; -%end; -proc http method="DELETE" url="&uri" &oauth_bearer; - headers -%if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" -%end; - "Accept"="*/*";/**/ -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 204 %then %do; - data _null_; infile &fname2; input; putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; -%else %put &sysmacroname: &path/&name successfully deleted; - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; -filename &fname1a clear; -libname &libref1a clear; - -%mend mv_deletejes; diff --git a/003_macros/mv_deleteviyafolder.sas b/003_macros/mv_deleteviyafolder.sas deleted file mode 100644 index a8e18a9..0000000 --- a/003_macros/mv_deleteviyafolder.sas +++ /dev/null @@ -1,154 +0,0 @@ -/*** HELP START ***//** - @file mv_deleteviyafolder.sas - @brief Creates a viya folder if that folder does not already exist - @details If not running in Studo 5 +, will expect an oauth token in a global - macro variable (default ACCESS_TOKEN). - - %mv_createfolder(path=/Public/test/blah) - %mv_deleteviyafolder(path=/Public/test) - - - @param [in] path= The full path of the folder to be deleted - @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to - contain the access token - @param [in] grant_type= (sas_services) Valid values are: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - @li sas_services - will use oauth_bearer=sas_services. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_existds.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_isblank.sas - -**//*** HELP END ***/ - -%macro mv_deleteviyafolder(path= - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=(%mf_isblank(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) -%mp_abort(iftrue=(%length(&path)=1) - ,mac=&sysmacroname - ,msg=%str(path value must be provided) -) - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%put &sysmacroname: fetching details for &path ; -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&path"; - %if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; - %end; -run; -%if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put &sysmacroname: Folder &path NOT FOUND - nothing to delete!; - %return; -%end; -%else %if &SYS_PROCHTTP_STATUS_CODE ne 200 %then %do; - /*data _null_;infile &fname1;input;putlog _infile_;run;*/ - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%put &sysmacroname: grab the follow on link ; -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; -data _null_; - set &libref1..links; - if rel='deleteRecursively' then - call symputx('href',quote("&base_uri"!!trim(href)),'l'); - else if rel='members' then - call symputx('mref',quote(cats("&base_uri",href,'?recursive=true')),'l'); -run; - -/* before we can delete the folder, we need to delete the children */ -%local fname1a; -%let fname1a=%mf_getuniquefileref(); -proc http method='GET' out=&fname1a &oauth_bearer - url=%unquote(%superq(mref)); -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; -%end; -run; -%put &=SYS_PROCHTTP_STATUS_CODE; -%local libref1a; -%let libref1a=%mf_getuniquelibref(); -libname &libref1a JSON fileref=&fname1a; - -%if %mf_existds(&libref1a..items_links) %then %do; - data _null_; - set &libref1a..items_links; - if href=:'/folders/folders' then return; - if rel='deleteResource' then - call execute('proc http method="DELETE" url=' - !!quote("&base_uri"!!trim(href)) - !!'; headers "Authorization"="Bearer &&&access_token_var" ' - !!' "Accept"="*/*";run; /**/'); - run; -%end; - -%put &sysmacroname: perform the delete operation ; -%local fname2; -%let fname2=%mf_getuniquefileref(); -proc http method='DELETE' out=&fname2 &oauth_bearer - url=%unquote(%superq(href)); - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - 'Accept'='*/*'; /**/ -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 204 %then %do; - data _null_; infile &fname2; input; putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; -%else %put &sysmacroname: &path successfully deleted; - -%if &mdebug=0 %then %do; - /* clear refs */ - filename &fname1 clear; - filename &fname2 clear; - libname &libref1 clear; -%end; - -%mend mv_deleteviyafolder; diff --git a/003_macros/mv_getclients.sas b/003_macros/mv_getclients.sas deleted file mode 100644 index be55a84..0000000 --- a/003_macros/mv_getclients.sas +++ /dev/null @@ -1,99 +0,0 @@ -/*** HELP START ***//** - @file mv_getclients.sas - @brief Get a list of Viya Clients - @details First, be sure you have an access token (which requires an app token). - - Using the macros here: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - An administrator needs to set you up with an access code: - - %mv_registerclient(outds=client) - - Navigate to the url from the log (opting in to the groups) and paste the - access code below: - - %mv_tokenauth(inds=client,code=wKDZYTEPK6) - - Now we can run the macro! - - %mv_getclients() - - @param [out] outds= (work.mv_getclients) - The library.dataset to be created that contains the list of clients - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_loc.sas - -**//*** HELP END ***/ - -%macro mv_getclients(outds=work.mv_getclients -)/*/STORE SOURCE*/; - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* first, get consul token needed to get client id / secret */ -data _null_; - infile "%mf_loc(VIYACONFIG)/etc/SASSecurityCertificateFramework/tokens/consul/default/client.token"; - input token:$64.; - call symputx('consul_token',token); -run; - -/* request the client details */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='POST' out=&fname1 - url="&base_uri/SASLogon/oauth/clients/consul?callback=false%str(&)serviceId=app"; - headers "X-Consul-Token"="&consul_token"; -run; - -%local libref1; -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; - -/* extract the token */ -data _null_; - set &libref1..root; - call symputx('access_token',access_token,'l'); -run; - -/* fetching folder details for provided path */ -%local fname2; -%let fname2=%mf_getuniquefileref(); -%let libref2=%mf_getuniquelibref(); - -proc http method='GET' out=&fname2 oauth_bearer=sas_services - url="&base_uri/SASLogon/oauth/clients"; - headers "Accept"="application/json"; -run; -/*data _null_;infile &fname1;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) -libname &libref2 JSON fileref=&fname1; - -data &outds; - set &libref2..items; -run; - - - -/* clear refs -filename &fname1 clear; -libname &libref1 clear; -*/ -%mend mv_getclients; diff --git a/003_macros/mv_getfoldermembers.sas b/003_macros/mv_getfoldermembers.sas deleted file mode 100644 index 9bc91e2..0000000 --- a/003_macros/mv_getfoldermembers.sas +++ /dev/null @@ -1,140 +0,0 @@ -/*** HELP START ***//** - @file - @brief Gets a list of folder members (and ids) for a given root - @details Returns all members for a particular Viya folder. Works at both root - level and below, and results are created in an output dataset. - - %mv_getfoldermembers(root=/Public, outds=work.mymembers) - - - @param [in] root= (/) The path for which to return the list of folders - @param [out] outds= (work.mv_getfolders) The output dataset to create. Format: - |ordinal_root|ordinal_items|creationTimeStamp| modifiedTimeStamp|createdBy|modifiedBy|id| uri|added| type|name|description| - |---|---|---|---|---|---|---|---|---|---|---|---| - |1|1|2021-05-25T11:15:04.204Z|2021-05-25T11:15:04.204Z|allbow|allbow|4f1e3945-9655-462b-90f2-c31534b3ca47|/folders/folders/ed701ff3-77e8-468d-a4f5-8c43dec0fd9e|2021-05-25T11:15:04.212Z|child|my_folder_name|My folder Description| - - @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to - contain the access token - @param [in] grant_type= (sas_services) Valid values are: - @li password - @li authorization_code - @li detect - @li sas_services - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_isblank.sas - -

Related Macros

- @li mv_createfolder.sas - @li mv_deletefoldermember.sas - @li mv_deleteviyafolder.sas - @li mv_getfoldermembers.test.sas - -**//*** HELP END ***/ - -%macro mv_getfoldermembers(root=/ - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,outds=mv_getfolders - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%if %mf_isblank(&root)=1 %then %let root=/; - -options noquotelenmax; - -/* request the client details */ -%local fname1 libref1; -%let fname1=%mf_getuniquefileref(); -%let libref1=%mf_getuniquelibref(); - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%if "&root"="/" %then %do; - /* if root just list root folders */ - proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/rootFolders?limit=1000"; - %if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; - %end; - run; - libname &libref1 JSON fileref=&fname1; - data &outds; - set &libref1..items; - run; -%end; -%else %do; - /* first get parent folder id */ - proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/folders/folders/@item?path=&root"; - %if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; - %end; - run; - /*data _null_;infile &fname1;input;putlog _infile_;run;*/ - libname &libref1 JSON fileref=&fname1; - /* now get the followon link to list members */ - %local href cnt; - %let cnt=0; - data _null_; - length rel href $512; - call missing(rel,href); - set &libref1..links; - if rel='members' then do; - url=cats("'","&base_uri",href,"?limit=10000'"); - call symputx('href',url,'l'); - call symputx('cnt',1,'l'); - end; - run; - %if &cnt=0 %then %do; - %put NOTE:;%put NOTE- No members found in &root!!;%put NOTE-; - %return; - %end; - %local fname2 libref2; - %let fname2=%mf_getuniquefileref(); - %let libref2=%mf_getuniquelibref(); - proc http method='GET' out=&fname2 &oauth_bearer - url=%unquote(%superq(href)); - %if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var"; - %end; - run; - libname &libref2 JSON fileref=&fname2; - data &outds; - length id $36 name $128 uri $64 type $32 description $256; - if _n_=1 then call missing (of _all_); - set &libref2..items; - run; - filename &fname2 clear; - libname &libref2 clear; -%end; - - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; - -%mend mv_getfoldermembers; diff --git a/003_macros/mv_getgroupmembers.sas b/003_macros/mv_getgroupmembers.sas deleted file mode 100644 index f0e7888..0000000 --- a/003_macros/mv_getgroupmembers.sas +++ /dev/null @@ -1,109 +0,0 @@ -/*** HELP START ***//** - @file mv_getgroupmembers.sas - @brief Creates a dataset with a list of group members - @details First, be sure you have an access token (which requires an app token) - - Using the macros here: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Now we can run the macro! - - %mv_getgroupmembers(All Users) - - outputs: - - ordinal_root num, - ordinal_items num, - version num, - id char(43), - name char(43), - providerId char(5), - implicit num - - @param [in] group Group id for which to return group members - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) - valid values are "password" or "authorization_code" (unquoted). - @param [out] outds= (work.viyagroupmembers) - The library.dataset to be created that contains the list of group members - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - -**//*** HELP END ***/ - -%macro mv_getgroupmembers(group - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,outds=work.viyagroupmembers - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -options noquotelenmax; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/identities/groups/&group/members?limit=10000"; - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - "Accept"="application/json"; -run; -/*data _null_;infile &fname1;input;putlog _infile_;run;*/ -%if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put NOTE: Group &group not found!!; - data &outds; - length id name $43; - call missing(of _all_); - run; -%end; -%else %do; - %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) - %let libref1=%mf_getuniquelibref(); - libname &libref1 JSON fileref=&fname1; - data &outds; - length id name $43; - set &libref1..items; - run; - libname &libref1 clear; -%end; - -/* clear refs */ -filename &fname1 clear; - -%mend mv_getgroupmembers; diff --git a/003_macros/mv_getgroups.sas b/003_macros/mv_getgroups.sas deleted file mode 100644 index ad7ff12..0000000 --- a/003_macros/mv_getgroups.sas +++ /dev/null @@ -1,86 +0,0 @@ -/*** HELP START ***//** - @file mv_getgroups.sas - @brief Creates a dataset with a list of viya groups - @details First, load the macros: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Next, execute: - - %mv_getgroups(outds=work.groups) - - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) - valid values are "password" or "authorization_code" (unquoted). - @param [out] outds= (work.viyagroups) - The library.dataset to be created that contains the list of groups - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - -**//*** HELP END ***/ - -%macro mv_getgroups(access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,outds=work.viyagroups - ); -%local oauth_bearer base_uri fname1 libref1; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -options noquotelenmax; -/* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%let fname1=%mf_getuniquefileref(); -%let libref1=%mf_getuniquelibref(); - -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/identities/groups?limit=10000"; - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - "Accept"="application/json"; -run; -/*data _null_;infile &fname1;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) -libname &libref1 JSON fileref=&fname1; - -data &outds; - set &libref1..items; -run; - - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; - -%mend mv_getgroups; diff --git a/003_macros/mv_getjobcode.sas b/003_macros/mv_getjobcode.sas deleted file mode 100644 index 9f21414..0000000 --- a/003_macros/mv_getjobcode.sas +++ /dev/null @@ -1,230 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract the source code from a SAS Viya Job - @details Extracts the SAS code from a Job into a fileref or physical file. - Example: - - %mv_getjobcode( - path=/Public/jobs - ,name=some_job - ,outfile=/tmp/some_job.sas - ) - - @param [in] access_token_var= The global macro variable to contain the access - token - @param [in] grant_type= valid values: - @li password - @liauthorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - @li sas_services - will use oauth_bearer=sas_services - @param [in] path= The SAS Drive path of the job - @param [in] name= The name of the job - @param [in] mdebug=(0) set to 1 to enable DEBUG messages - @param [out] outref=(0) A fileref to which to write the source code (will be - created with a TEMP engine) - @param [out] outfile=(0) A file to which to write the source code - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mv_getfoldermembers.sas - -**//*** HELP END ***/ - -%macro mv_getjobcode(outref=0,outfile=0 - ,name=0,path=0 - ,contextName=SAS Job Execution compute context - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ); -%local dbg bufsize varcnt fname1 fname2 errmsg; -%if &mdebug=1 %then %do; - %put &sysmacroname local entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -%mp_abort(iftrue=("&path"="0") - ,mac=&sysmacroname - ,msg=%str(Job Path not provided) -) -%mp_abort(iftrue=("&name"="0") - ,mac=&sysmacroname - ,msg=%str(Job Name not provided) -) -%mp_abort(iftrue=("&outfile"="0" and "&outref"="0") - ,mac=&sysmacroname - ,msg=%str(Output destination (file or fileref) must be provided) -) -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); -data;run; -%local foldermembers; -%let foldermembers=&syslast; -%mv_getfoldermembers(root=&path - ,access_token_var=&access_token_var - ,grant_type=&grant_type - ,outds=&foldermembers -) -%local joburi; -%let joburi=0; -data _null_; - length name uri $512; - call missing(name,uri); - set &foldermembers; - if name="&name" and uri=:'/jobDefinitions/definitions' - then call symputx('joburi',uri); -run; -%mp_abort(iftrue=("&joburi"="0") - ,mac=&sysmacroname - ,msg=%str(Job &path/&name not found) -) - -/* prepare request*/ -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&joburi"; - headers "Accept"="application/vnd.sas.job.definition+json" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fname1; - input; - putlog _infile_; - run; -%end; - -%mp_abort( - iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -%let fname2=%mf_getuniquefileref(); -filename &fname2 temp ; - -/* cannot use lua IO package as not available in Viya 4 */ -/* so use data step to read the JSON until the string `"code":"` is found */ -data _null_; - file &fname2 recfm=n; - infile &fname1 lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - retain startwrite 0; - if startwrite=0 and sourcechar='"' then do; - reentry: - input sourcechar $ 1. @@; - if sourcechar='c' then do; - reentry2: - input sourcechar $ 1. @@; - if sourcechar='o' then do; - input sourcechar $ 1. @@; - if sourcechar='d' then do; - input sourcechar $ 1. @@; - if sourcechar='e' then do; - input sourcechar $ 1. @@; - if sourcechar='"' then do; - input sourcechar $ 1. @@; - if sourcechar=':' then do; - input sourcechar $ 1. @@; - if sourcechar='"' then do; - putlog 'code found'; - startwrite=1; - input sourcechar $ 1. @@; - end; - end; - else if sourcechar='c' then goto reentry2; - end; - end; - else if sourcechar='"' then goto reentry; - end; - else if sourcechar='"' then goto reentry; - end; - else if sourcechar='"' then goto reentry; - end; - else if sourcechar='"' then goto reentry; - end; - /* once the `"code":"` string is found, write until unescaped `"` is found */ - if startwrite=1 then do; - if sourcechar='\' then do; - input sourcechar $ 1. @@; - if sourcechar in ('"','\') then put sourcechar char1.; - else if sourcechar='n' then put '0A'x; - else if sourcechar='r' then put '0D'x; - else if sourcechar='t' then put '09'x; - else if sourcechar='u' then do; - length uni $4; - input uni $ 4. @@; - sourcechar=unicode('\u'!!uni); - put sourcechar char1.; - end; - else do; - call symputx('errmsg',"Uncaught escape char: "!!sourcechar,'l'); - call symputx('syscc',99); - stop; - end; - end; - else if sourcechar='"' then stop; - else put sourcechar char1.; - end; -run; - -%mp_abort(iftrue=("&syscc"="99") - ,mac=mv_getjobcode - ,msg=%str(&errmsg) -) - -/* export to desired destination */ -%if "&outref"="0" %then %do; - data _null_; - file "&outfile" lrecl=32767; -%end; -%else %do; - filename &outref temp; - data _null_; - file &outref; -%end; - infile &fname2; - input; - put _infile_; - &dbg. putlog _infile_; -run; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%else %do; - /* clear refs */ - filename &fname1 clear; - filename &fname2 clear; -%end; - -%mend mv_getjobcode; diff --git a/003_macros/mv_getjoblog.sas b/003_macros/mv_getjoblog.sas deleted file mode 100644 index 40e8cc4..0000000 --- a/003_macros/mv_getjoblog.sas +++ /dev/null @@ -1,290 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract the log from a completed SAS Viya Job - @details Extracts log from a Viya job and writes it out to a fileref. - - To query the job, you need the URI. Sample code for achieving this - is provided below. - - ## Example - - %* First, compile the macros; - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* Next, create a job (in this case, a web service); - filename ft15f001 temp; - parmcards4; - data ; - rand=ranuni(0)*1000; - do x=1 to rand; - y=rand*4; - output; - end; - run; - proc sort data=&syslast - by descending y; - run; - ;;;; - %mv_createwebservice(path=/Public/temp,name=demo) - - %* Execute it; - %mv_jobexecute(path=/Public/temp - ,name=demo - ,outds=work.info - ) - - %* Wait for it to finish; - data work.info; - set work.info; - where method='GET' and rel='state'; - run; - %mv_jobwaitfor(ALL,inds=work.info,outds=work.jobstates) - - %* and grab the uri; - data _null_; - set work.jobstates; - call symputx('uri',uri); - run; - - %* Finally, fetch the log; - %mv_getjoblog(uri=&uri,outref=mylog) - - This macro is used by the mv_jobwaitfor.sas macro, which is generally a more - convenient way to wait for the job to finish before fetching the log. - - If the remote session calls `endsas` then it is not possible to get the log - from the provided uri, and so the log from the parent session is fetched - instead. This happens for a 400 response, eg below: - - ErrorResponse[version=2,status=400,err=5113,id=,message=The session - requested is currently in a failed or stopped state.,detail=[path: - /compute/sessions/LONGURI-ses0006/jobs/LONGURI/log/content, traceId: 63 - 51aa617d01fd2b],remediation=Correct the errors in the session request, - and create a new session.,targetUri=,errors=[],links=[]] - - @param [in] access_token_var= The global macro variable to contain the access - token - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [in] grant_type= valid values: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - @li sas_services - will use oauth_bearer=sas_services. - @param [in] uri= The uri of the running job for which to fetch the status, - in the format `/jobExecution/jobs/$UUID` (unquoted). - @param [out] outref= The output fileref to which to APPEND the log (is always - appended). - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_existfileref.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - -**//*** HELP END ***/ - -%macro mv_getjoblog(uri=0,outref=0 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ); -%local dbg libref1 libref2 loglocation fname1 fname2; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -/* validation in datastep for better character safety */ -%local errmsg errflg; -data _null_; - uri=symget('uri'); - if length(uri)<12 then do; - call symputx('errflg',1); - call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); - end; - if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do; - call symputx('errflg',1); - call symputx('errmsg', - "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" - !!" but is actually like:"!!uri,'l'); - end; -run; - -%mp_abort(iftrue=(&errflg=1) - ,mac=&sysmacroname - ,msg=%str(&errmsg) -) - -%mp_abort(iftrue=(&outref=0) - ,mac=&sysmacroname - ,msg=%str(Output fileref should be provided) -) - -%if %mf_existfileref(&outref) ne 1 %then %do; - filename &outref temp; -%end; - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* prepare request*/ -%let fname1=%mf_getuniquefileref(); -%let fname2=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&uri"; - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; -%if &mdebug=1 %then %do; - %put &sysmacroname: fetching log loc from &uri; - data _null_;infile &fname1;input;putlog _infile_;run; -%end; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; - data _null_;infile &fname1;input;putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%let libref1=%mf_getuniquelibref(); -libname &libref1 JSON fileref=&fname1; -data _null_; - set &libref1..root; - call symputx('loglocation',loglocation,'l'); -run; - -/* validate log path*/ -%let errflg=1; -%let errmsg=No loglocation entry in &fname1 fileref; -data _null_; - uri=symget('loglocation'); - if length(uri)<12 then do; - call symputx('errflg',1); - call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); - end; - else if (scan(uri,1,'/') ne 'compute' or scan(uri,2,'/') ne 'sessions') - and (scan(uri,1,'/') ne 'files' or scan(uri,2,'/') ne 'files') - then do; - call symputx('errflg',1); - call symputx('errmsg', - "URI should be in format /compute/sessions/$$$$UUID$$$$/jobs/$$$$UUID$$$$" - !!" or /files/files/$$$$UUID$$$$" - !!" but is actually like:"!!uri,'l'); - end; - else do; - call symputx('errflg',0,'l'); - call symputx('logloc',uri,'l'); - end; -run; - -%mp_abort(iftrue=(%str(&errflg)=1) - ,mac=&sysmacroname - ,msg=%str(&errmsg) -) - -/* we have a log uri - now fetch the log */ -%&dbg.put &sysmacroname: querying &base_uri&logloc/content; -proc http method='GET' out=&fname2 &oauth_bearer - url="&base_uri&logloc/content?limit=10000"; - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; - -%if &mdebug=1 %then %do; - %put &sysmacroname: fetching log content from &base_uri&logloc/content; - data _null_;infile &fname2;input;putlog _infile_;run; -%end; - -%if &SYS_PROCHTTP_STATUS_CODE=400 %then %do; - /* fetch log from parent session */ - %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1); - %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content; - proc http method='GET' out=&fname2 &oauth_bearer - url="&base_uri&logloc/log/content?limit=10000"; - headers - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; - run; - %if &mdebug=1 %then %do; - %put &sysmacroname: fetching log content from &base_uri&logloc/log/content; - data _null_;infile &fname2;input;putlog _infile_;run; - %end; -%end; - -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 -%then %do; - %if &mdebug ne 1 %then %do; /* have already output above */ - data _null_;infile &fname2;input;putlog _infile_;run; - %end; - %mp_abort(mac=&sysmacroname - ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%let libref2=%mf_getuniquelibref(); -libname &libref2 JSON fileref=&fname2; -data _null_; - file &outref mod; - if _n_=1 then do; - put "/** SASJS Viya Job Log Extract start: &uri **/"; - end; - set &libref2..items end=last; - %if &mdebug=1 %then %do; - putlog line; - %end; - put line; - if last then do; - put "/** SASJS Viya Job Log Extract end: &uri **/"; - end; -run; - -%if &mdebug=0 %then %do; - filename &fname1 clear; - filename &fname2 clear; - libname &libref1 clear; - libname &libref2 clear; -%end; -%else %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%mend mv_getjoblog; - - - diff --git a/003_macros/mv_getjobresult.sas b/003_macros/mv_getjobresult.sas deleted file mode 100644 index a830482..0000000 --- a/003_macros/mv_getjobresult.sas +++ /dev/null @@ -1,232 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract the result from a completed SAS Viya Job - @details Extracts result from a Viya job and writes it out to a fileref - and/or a JSON-engine library. - - To query the job, you need the URI. Sample code for achieving this - is provided below. - - ## Example - - First, compile the macros: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Next, create a job (in this case, a web service): - - filename ft15f001 temp; - parmcards4; - data test; - rand=ranuni(0)*1000; - do x=1 to rand; - y=rand*4; - output; - end; - run; - proc sort data=&syslast - by descending y; - run; - %webout(OPEN) - %webout(OBJ, test) - %webout(CLOSE) - ;;;; - %mv_createwebservice(path=/Public/temp,name=demo) - - Execute it: - - %mv_jobexecute(path=/Public/temp - ,name=demo - ,outds=work.info - ) - - Wait for it to finish, and grab the uri: - - data _null_; - set work.info; - if method='GET' and rel='self'; - call symputx('uri',uri); - run; - - Finally, fetch the result (In this case, WEBOUT): - - %mv_getjobresult(uri=&uri,result=WEBOUT_JSON,outref=myweb,outlib=myweblib) - - - @param [in] access_token_var= The global macro variable containing the access - token - @param [in] mdebug= set to 1 to enable DEBUG messages - @param [in] grant_type= valid values: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - @li sas_services - will use oauth_bearer=sas_services. - @param [in] uri= The uri of the running job for which to fetch the status, - in the format `/jobExecution/jobs/$UUID` (unquoted). - - @param [out] result= (WEBOUT_JSON) The result type to capture. Resolves - to "_[column name]" from the results table when parsed with the JSON libname - engine. Example values: - @li WEBOUT_JSON - @li WEBOUT_TXT - - @param [out] outref= (0) The output fileref to which to write the results - @param [out] outlib= (0) The output library to which to assign the results - (assumes the data is in JSON format) - - - @version VIYA V.03.05 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mp_binarycopy.sas - @li mf_getplatform.sas - @li mf_existfileref.sas - -**//*** HELP END ***/ - -%macro mv_getjobresult(uri=0 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,mdebug=0 - ,result=WEBOUT_JSON - ,outref=0 - ,outlib=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - - -/* validation in datastep for better character safety */ -%local errmsg errflg; -data _null_; - uri=symget('uri'); - if length(uri)<12 then do; - call symputx('errflg',1); - call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); - end; - if scan(uri,-1)='state' or scan(uri,1) ne 'jobExecution' then do; - call symputx('errflg',1); - call symputx('errmsg', - "URI should be in format /jobExecution/jobs/$$$$UUID$$$$" - !!" but is actually like: &uri",'l'); - end; -run; - -%mp_abort(iftrue=(&errflg=1) - ,mac=&sysmacroname - ,msg=%str(&errmsg) -) - -%if &outref ne 0 and %mf_existfileref(&outref) ne 1 %then %do; - filename &outref temp; -%end; - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetch job info */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri&uri"; - headers "Accept"="application/json" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; - data _null_;infile &fname1;input;putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; -%if &mdebug=1 %then %do; - data _null_; - infile &fname1 lrecl=32767; - input; - putlog _infile_; - run; -%end; - -/* extract results link */ -%local lib1 resuri; -%let lib1=%mf_getuniquelibref(); -libname &lib1 JSON fileref=&fname1; -data _null_; - set &lib1..results; - call symputx('resuri',_&result,'l'); - &dbg putlog "&sysmacroname results: " (_all_)(=); -run; -%mp_abort(iftrue=("&resuri"=".") - ,mac=&sysmacroname - ,msg=%str(Variable _&result did not exist in the response json) -) - -/* extract results */ -%local fname2; -%let fname2=%mf_getuniquefileref(); -proc http method='GET' out=&fname2 &oauth_bearer - url="&base_uri&resuri/content?limit=10000"; - headers "Accept"="application/json" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; -%if &mdebug=1 %then %do; - /* send one char at a time as the json can be very wide */ - data _null_; - infile &fname2 recfm=n; - input char $char1. ; - putlog char $char1. @; - run; -%end; - -%if &outref ne 0 %then %do; - filename &outref temp; - %mp_binarycopy(inref=&fname2,outref=&outref) -%end; -%if &outlib ne 0 %then %do; - libname &outlib JSON fileref=&fname2; -%end; - -%if &mdebug=0 %then %do; - filename &fname1 clear; - filename &fname2 clear; - libname &lib1 clear; -%end; -%else %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; - -%mend mv_getjobresult; diff --git a/003_macros/mv_getjobstate.sas b/003_macros/mv_getjobstate.sas deleted file mode 100644 index 18a602a..0000000 --- a/003_macros/mv_getjobstate.sas +++ /dev/null @@ -1,171 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract the status from a running SAS Viya job - @details Extracts the status from a running job and appends it to an output - dataset with the following structure: - - | uri | state | timestamp | - |---------------------------------------------------------------|---------|--------------------| - | /jobExecution/jobs/5cebd840-2063-42c1-be0c-421ec3e1c175/state | running | 15JAN2021:12:35:08 | - - To query the running job, you need the URI. Sample code for achieving this - is provided below. - - ## Example - - First, compile the macros: - - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Next, create a long running job (in this case, a web service): - - filename ft15f001 temp; - parmcards4; - data ; - rand=ranuni(0)*1000; - do x=1 to rand; - y=rand*4; - output; - end; - run; - data _null_; - call sleep(5,1); - run; - ;;;; - %mv_createwebservice(path=/Public/temp,name=demo) - - Execute it, grab the uri, and finally, check the job status: - - %mv_jobexecute(path=/Public/temp - ,name=demo - ,outds=work.info - ) - - data _null_; - set work.info; - if method='GET' and rel='state'; - call symputx('uri',uri); - run; - - %mv_getjobstate(uri=&uri,outds=results) - - You can run this macro as part of a loop to await the final 'completed' - status. - The full list of status values is: - - @li idle - @li pending - @li running - @li canceled - @li completed - @li failed - - If you have one or more jobs that you'd like to wait for completion you can - also use the [mv_jobwaitfor](/mv__jobwaitfor_8sas.html) macro. - - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= valid values: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. - @li sas_services - will use oauth_bearer=sas_services. - @param [in] uri= The uri of the running job for which to fetch the status, - in the format `/jobExecution/jobs/$UUID/state` (unquoted). - @param [out] outds= The output dataset in which to APPEND the status. Three - fields are appended: `CHECK_TM`, `URI` and `STATE`. If the dataset does not - exist, it is created. - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - -**//*** HELP END ***/ - -%macro mv_getjobstate(uri=0,outds=work.mv_getjobstate - ,contextName=SAS Job Execution compute context - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -/* validation in datastep for better character safety */ -%local errmsg errflg; -data _null_; - uri=symget('uri'); - if length(uri)<12 then do; - call symputx('errflg',1); - call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l'); - end; - if scan(uri,-1) ne 'state' or scan(uri,1) ne 'jobExecution' then do; - - call symputx('errflg',1); - call symputx('errmsg', - "URI should be in format /jobExecution/jobs/$$$$UUID$$$$/state" - !!" but is actually like: &uri",'l'); - end; -run; - -%mp_abort(iftrue=(&errflg=1) - ,mac=&sysmacroname - ,msg=%str(&errmsg) -) - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%local fname0; -%let fname0=%mf_getuniquefileref(); - -proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&uri"; - headers "Accept"="text/plain" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; ; -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; - data _null_;infile &fname0;input;putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -data; - format uri $128. state $32. timestamp datetime19.; - infile &fname0; - uri="&uri"; - timestamp=datetime(); - input; - state=_infile_; -run; - -proc append base=&outds data=&syslast; -run; - -filename &fname0 clear; - -%mend mv_getjobstate; diff --git a/003_macros/mv_getusergroups.sas b/003_macros/mv_getusergroups.sas deleted file mode 100644 index 9e5c75c..0000000 --- a/003_macros/mv_getusergroups.sas +++ /dev/null @@ -1,95 +0,0 @@ -/*** HELP START ***//** - @file mv_getusergroups.sas - @brief Creates a dataset with a list of groups for a particular user - @details If using outside of Viya SPRE, then an access token is needed. - - Compile the macros here: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Then run the macro! - - %mv_getusergroups(&sysuserid,outds=users) - - @param [in] user The username for which to return the list of groups - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) - Valid values are "password" or "authorization_code" (unquoted). - @param [out] outds= (work.mv_getusergroups) - The library.dataset to be created that contains the list of groups - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - -**//*** HELP END ***/ - -%macro mv_getusergroups(user - ,outds=work.mv_getusergroups - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; -%put &sysmacroname: grant_type=&grant_type; -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) -options noquotelenmax; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -%let libref1=%mf_getuniquelibref(); - -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/identities/users/&user/memberships?limit=10000"; - headers -%if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" -%end; - "Accept"="application/json"; -run; -/*data _null_;infile &fname1;input;putlog _infile_;run;*/ -%if &SYS_PROCHTTP_STATUS_CODE=404 %then %do; - %put NOTE: User &user not found!!; -%end; -%else %do; - %mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; -libname &libref1 JSON fileref=&fname1; - -data &outds; - set &libref1..items; -run; - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; - -%mend mv_getusergroups; diff --git a/003_macros/mv_getusers.sas b/003_macros/mv_getusers.sas deleted file mode 100644 index 982f981..0000000 --- a/003_macros/mv_getusers.sas +++ /dev/null @@ -1,118 +0,0 @@ -/*** HELP START ***//** - @file mv_getusers.sas - @brief Creates a dataset with a list of users - @details First, be sure you have an access token (which requires an app token) - - Using the macros here: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - An administrator needs to set you up with an access code: - - %mv_registerclient(outds=client) - - Navigate to the url from the log (opting in to the groups) and paste the - access code below: - - %mv_tokenauth(inds=client,code=wKDZYTEPK6) - - Now we can run the macro! - - %mv_getusers(outds=users) - - Output (lengths are dynamic): - - ordinal_root num, - ordinal_items num, - version num, - id char(20), - name char(23), - providerId char(4), - type char(4), - creationTimeStamp char(24), - modifiedTimeStamp char(24), - state char(6) - - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] grant_type= (sas_services) Valid values: - * password - * authorization_code - * detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. - * sas_services - will use oauth_bearer=sas_services - - @param [out] outds= (work.mv_getusers) - The library.dataset to be created that contains the list of groups - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - -**//*** HELP END ***/ - -%macro mv_getusers(outds=work.mv_getusers - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ); -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; -%put &sysmacroname: grant_type=&grant_type; -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -options noquotelenmax; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* fetching folder details for provided path */ -%local fname1; -%let fname1=%mf_getuniquefileref(); -%let libref1=%mf_getuniquelibref(); - -proc http method='GET' out=&fname1 &oauth_bearer - url="&base_uri/identities/users?limit=10000"; -%if &grant_type=authorization_code %then %do; - headers "Authorization"="Bearer &&&access_token_var" - "Accept"="application/json"; -%end; -%else %do; - headers "Accept"="application/json"; -%end; -run; -/*data _null_;infile &fname1;input;putlog _infile_;run;*/ -%mp_abort(iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) -libname &libref1 JSON fileref=&fname1; - -data &outds; - set &libref1..items; -run; - -/* clear refs */ -filename &fname1 clear; -libname &libref1 clear; - -%mend mv_getusers; diff --git a/003_macros/mv_jobexecute.sas b/003_macros/mv_jobexecute.sas deleted file mode 100644 index 8515012..0000000 --- a/003_macros/mv_jobexecute.sas +++ /dev/null @@ -1,188 +0,0 @@ -/*** HELP START ***//** - @file - @brief Executes a SAS Viya Job - @details Triggers a SAS Viya Job, with optional URL parameters, using - the JES web app. - - First, compile the macros: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Then, execute the job! - - %mv_jobexecute(path=/Public/folder - ,name=somejob - ) - - Example with parameters: - - %mv_jobexecute(path=/Public/folder - ,name=somejob - ,paramstring=%str("macvarname":"macvarvalue","answer":42) - ) - - @param [in] access_token_var= The global macro variable to contain the access - token - @param [in] grant_type= valid values: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - @li sas_services - will use oauth_bearer=sas_services - - @param [in] path= The SAS Drive path to the job being executed - @param [in] name= The name of the job to execute - @param [in] paramstring= A JSON fragment with name:value pairs, eg: - `"name":"value"` or "name":"value","name2":42`. This will need to be - wrapped in `%str()`. - - @param [in] contextName= Context name with which to run the job. - Default = `SAS Job Execution compute context` - @param [in] mdebug= set to 1 to enable DEBUG messages - @param [out] outds= (work.mv_jobexecute) The output dataset containing links - - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mv_getfoldermembers.sas - -**//*** HELP END ***/ - -%macro mv_jobexecute(path=0 - ,name=0 - ,contextName=SAS Job Execution compute context - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,paramstring=0 - ,outds=work.mv_jobexecute - ,mdebug=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort(iftrue=("&path"="0") - ,mac=&sysmacroname - ,msg=%str(Path not provided) -) -%mp_abort(iftrue=("&name"="0") - ,mac=&sysmacroname - ,msg=%str(Job Name not provided) -) - -options noquotelenmax; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -data;run; -%local foldermembers; -%let foldermembers=&syslast; -%mv_getfoldermembers(root=&path - ,access_token_var=&access_token_var - ,grant_type=&grant_type - ,outds=&foldermembers -) - -%local joburi; -%let joburi=0; -data _null_; - length name uri $512; - call missing(name,uri); - set &foldermembers; - if name="&name" and uri=:'/jobDefinitions/definitions' - then call symputx('joburi',uri); -run; - -%mp_abort(iftrue=("&joburi"="0") - ,mac=&sysmacroname - ,msg=%str(Job &path/&name not found) -) - -/* prepare request*/ -%local fname0 fname1; -%let fname0=%mf_getuniquefileref(); -%let fname1=%mf_getuniquefileref(); - -data _null_; - file &fname0; - length joburi contextname $128 paramstring $32765; - joburi=quote(trim(symget('joburi'))); - contextname=quote(trim(symget('contextname'))); - _program=quote("&path/&name"); - paramstring=symget('paramstring'); - put '{"jobDefinitionUri":' joburi ; - put ' ,"arguments":{"_contextName":' contextname; - put ' ,"_program":' _program; - if paramstring ne "0" then do; - put ' ,' paramstring; - end; - put '}}'; -run; - -proc http method='POST' in=&fname0 out=&fname1 &oauth_bearer - url="&base_uri/jobExecution/jobs"; - headers "Content-Type"="application/vnd.sas.job.execution.job.request+json" - "Accept"="application/vnd.sas.job.execution.job+json" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; - ; -run; -%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then -%do; - data _null_;infile &fname0;input;putlog _infile_;run; - data _null_;infile &fname1;input;putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) -%end; - -%local libref; -%let libref=%mf_getuniquelibref(); -libname &libref JSON fileref=&fname1; - -data &outds; - set &libref..links; - _program="&path/&name"; -run; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%else %do; - /* clear refs */ - filename &fname0 clear; - filename &fname1 clear; - libname &libref; -%end; -%mend mv_jobexecute; diff --git a/003_macros/mv_jobflow.sas b/003_macros/mv_jobflow.sas deleted file mode 100644 index 7de858d..0000000 --- a/003_macros/mv_jobflow.sas +++ /dev/null @@ -1,395 +0,0 @@ -/*** HELP START ***//** - @file - @brief Execute a series of job flows - @details Very (very) simple flow manager. Jobs execute in sequential waves, - all previous waves must finish successfully. - - The input table is formed as per below. Each observation represents one job. - Each variable is converted into a macro variable with the same name. - - ## Input table (minimum variables needed) - - @li _PROGRAM - Provides the path to the job itself - @li FLOW_ID - Numeric value, provides sequential ordering capability. Is - optional, will default to 0 if not provided. - @li _CONTEXTNAME - Dictates which context should be used to run the job. If - blank, or not provided, will default to `SAS Job Execution compute context`. - - Any additional variables provided in this table are converted into macro - variables and passed into the relevant job. - - |_PROGRAM| FLOW_ID (optional)| _CONTEXTNAME (optional) | - |---|---|---| - |/Public/jobs/somejob1|0|SAS Job Execution compute context| - |/Public/jobs/somejob2|0|SAS Job Execution compute context| - - ## Output table (minimum variables produced) - - @li _PROGRAM - the SAS Drive path of the job - @li URI - the URI of the executed job - @li STATE - the completed state of the job - @li TIMESTAMP - the datetime that the job completed - @li JOBPARAMS - the parameters that were passed to the job - @li FLOW_ID - the id of the flow in which the job was executed - - ![https://i.imgur.com/nZE9PvT.png](https://i.imgur.com/nZE9PvT.png) - - To avoid hammering the box with many hits in rapid succession, a one - second pause is made between every request. - - - ## Example - - First, compile the macros: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Next, create some jobs (in this case, as web services): - - filename ft15f001 temp; - parmcards4; - %put this is job: &_program; - %put this was run in flow &flow_id; - data ; - rand=ranuni(0)*¯ovar1; - do x=1 to rand; - y=rand*¯ovar2; - if y=100 then abort; - output; - end; - run; - ;;;; - %mv_createwebservice(path=/Public/temp,name=demo1) - %mv_createwebservice(path=/Public/temp,name=demo2) - - Prepare an input table with 60 executions: - - data work.inputjobs; - _contextName='SAS Job Execution compute context'; - do flow_id=1 to 3; - do i=1 to 20; - _program='/Public/temp/demo1'; - macrovar1=10*i; - macrovar2=4*i; - output; - i+1; - _program='/Public/temp/demo2'; - macrovar1=40*i; - macrovar2=44*i; - output; - end; - end; - run; - - Trigger the flow - - %mv_jobflow(inds=work.inputjobs - ,maxconcurrency=4 - ,outds=work.results - ,outref=myjoblog - ) - - data _null_; - infile myjoblog; - input; put _infile_; - run; - - - @param [in] access_token_var= The global macro variable to contain the - access token - @param [in] grant_type= valid values: - @li password - @li authorization_code - @li detect - will check if access_token exists, if not will use - sas_services if a SASStudioV session else authorization_code. Default - option. - @li sas_services - will use oauth_bearer=sas_services - @param [in] inds= The input dataset containing a list of jobs and parameters - @param [in] maxconcurrency= The max number of parallel jobs to run. Default=8. - @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete - succcessfully - @param [in] mdebug= set to 1 to enable DEBUG messages - @param [out] outds= The output dataset containing the results - @param [out] outref= The output fileref to which to append the log file(s). - - @version VIYA V.03.05 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mf_nobs.sas - @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_existvarlist.sas - @li mv_jobwaitfor.sas - @li mv_jobexecute.sas - -**//*** HELP END ***/ - -%macro mv_jobflow(inds=0,outds=work.mv_jobflow - ,maxconcurrency=8 - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,outref=0 - ,raise_err=0 - ,mdebug=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; - %put inds vars:; - data _null_; - set &inds; - putlog (_all_)(=); - run; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort(iftrue=("&inds"="0") - ,mac=&sysmacroname - ,msg=%str(Input dataset was not provided) -) -%mp_abort(iftrue=(%mf_existVarList(&inds,_PROGRAM)=0) - ,mac=&sysmacroname - ,msg=%str(The _PROGRAM column must exist on input dataset &inds) -) -%mp_abort(iftrue=(&maxconcurrency<1) - ,mac=&sysmacroname - ,msg=%str(The maxconcurrency variable should be a positive integer) -) - -/* set defaults if not provided */ -%if %mf_existVarList(&inds,_CONTEXTNAME FLOW_ID)=0 %then %do; - data &inds; - %if %mf_existvarList(&inds,_CONTEXTNAME)=0 %then %do; - length _CONTEXTNAME $128; - retain _CONTEXTNAME "SAS Job Execution compute context"; - %end; - %if %mf_existvarList(&inds,FLOW_ID)=0 %then %do; - retain FLOW_ID 0; - %end; - set &inds; - &dbg. putlog (_all_)(=); - run; -%end; - -%local missings; -proc sql noprint; -select count(*) into: missings - from &inds - where flow_id is null or _program is null; -%mp_abort(iftrue=(&missings>0) - ,mac=&sysmacroname - ,msg=%str(input dataset has &missings missing values for FLOW_ID or _PROGRAM) -) - -%if %mf_nobs(&inds)=0 %then %do; - %put No observations in &inds! Leaving macro &sysmacroname; - %return; -%end; - -/* ensure output table is available */ -data &outds;run; -proc sql; -drop table &outds; - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - - -/* get flows */ -proc sort data=&inds; - by flow_id; -run; -data _null_; - set &inds (keep=flow_id) end=last; - by flow_id; - if last.flow_id then do; - cnt+1; - call symputx(cats('flow',cnt),flow_id,'l'); - end; - if last then call symputx('flowcnt',cnt,'l'); -run; - -/* prepare temporary datasets and frefs */ -%local fid jid jds jjson jdsapp jdsrunning jdswaitfor jfref; -data;run;%let jds=&syslast; -data;run;%let jjson=&syslast; -data;run;%let jdsapp=&syslast; -data;run;%let jdsrunning=&syslast; -data;run;%let jdswaitfor=&syslast; -%let jfref=%mf_getuniquefileref(); - -/* start loop */ -%do fid=1 %to &flowcnt; - - %if not ( &raise_err and &syscc ) %then %do; - - %put preparing job attributes for flow &&flow&fid; - %local jds jcnt; - data &jds(drop=_contextName _program); - set &inds(where=(flow_id=&&flow&fid)); - if _contextName='' then _contextName="SAS Job Execution compute context"; - call symputx(cats('job',_n_),_program,'l'); - call symputx(cats('context',_n_),_contextName,'l'); - call symputx('jcnt',_n_,'l'); - &dbg. if _n_= 1 then putlog "Loop &fid"; - &dbg. putlog (_all_)(=); - run; - %put exporting job variables in json format; - %do jid=1 %to &jcnt; - data &jjson; - set &jds; - if _n_=&jid then do; - output; - stop; - end; - run; - proc json out=&jfref; - export &jjson / nosastags fmtnumeric; - run; - data _null_; - infile &jfref lrecl=32767; - input; - jparams=cats('jparams',symget('jid')); - call symputx(jparams,substr(_infile_,3,length(_infile_)-4)); - run; - %local jobuid&jid; - %let jobuid&jid=0; /* used in next loop */ - %end; - %local concurrency completed; - %let concurrency=0; - %let completed=0; - proc sql; drop table &jdsrunning; - %do jid=1 %to &jcnt; - /** - * now we can execute the jobs up to the maxconcurrency setting - */ - %if "&&job&jid" ne "0" %then %do; /* this var is zero if job finished */ - - /* check to see if the job finished in the previous round */ - %if %sysfunc(exist(&outds))=1 %then %do; - %local jobcheck; %let jobcheck=0; - proc sql noprint; - select count(*) into: jobcheck - from &outds where uuid="&&jobuid&jid"; - %if &jobcheck>0 %then %do; - %put &&job&jid in flow &fid with uid &&jobuid&jid completed!; - %let job&jid=0; - %end; - %end; - - /* check if job was triggered and, if - so, if we have enough slots to run? */ - %if ("&&jobuid&jid"="0") and (&concurrency<&maxconcurrency) %then %do; - - /* But only start if no issues detected so far */ - %if not ( &raise_err and &syscc ) %then %do; - - %local jobname jobpath; - %let jobname=%scan(&&job&jid,-1,/); - %let jobpath= - %substr(&&job&jid,1,%length(&&job&jid)-%length(&jobname)-1); - - %put executing &jobpath/&jobname with paramstring &&jparams&jid; - %mv_jobexecute(path=&jobpath - ,name=&jobname - ,paramstring=%superq(jparams&jid) - ,outds=&jdsapp - ,contextname=&&context&jid - ) - data &jdsapp; - format jobparams $32767.; - set &jdsapp(where=(method='GET' and rel='state')); - jobparams=symget("jparams&jid"); - /* uri here has the /state suffix */ - uuid=scan(uri,-2,'/'); - call symputx("jobuid&jid",uuid,'l'); - run; - proc append base=&jdsrunning data=&jdsapp; - run; - %let concurrency=%eval(&concurrency+1); - /* sleep one second after every request to smooth the impact */ - data _null_; - call sleep(1,1); - run; - - %end; - %else %do; /* Job was skipped due to problems */ - - %put jobid &&job&jid in flow &fid skipped due to SYSCC (&syscc); - %let completed = %eval(&completed+1); - %let job&jid=0; /* Indicate job has finished */ - - %end; - - %end; - %end; - %if &jid=&jcnt %then %do; - /* we are at the end of the loop - check which jobs have finished */ - %mv_jobwaitfor(ANY,inds=&jdsrunning,outds=&jdswaitfor,outref=&outref - ,raise_err=&raise_err,mdebug=&mdebug) - %local done; - %let done=%mf_nobs(&jdswaitfor); - %if &done>0 %then %do; - %let completed=%eval(&completed+&done); - %let concurrency=%eval(&concurrency-&done); - data &jdsapp; - set &jdswaitfor; - flow_id=&&flow&fid; - uuid=scan(uri,-1,'/'); - run; - proc append base=&outds data=&jdsapp; - run; - %end; - proc sql; - delete from &jdsrunning - where uuid in (select uuid from &outds - where state in ('canceled','completed','failed') - ); - - /* loop again if jobs are left */ - %if &completed < &jcnt %then %do; - %let jid=0; - %put looping flow &fid again; - %put &completed of &jcnt jobs completed, &concurrency jobs running; - %end; - %end; - %end; - - %end; - %else %do; - - %put Flow &&flow&fid skipped due to SYSCC (&syscc); - - %end; - /* back up and execute the next flow */ -%end; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; - -%mend mv_jobflow; diff --git a/003_macros/mv_jobwaitfor.sas b/003_macros/mv_jobwaitfor.sas deleted file mode 100644 index 21896ed..0000000 --- a/003_macros/mv_jobwaitfor.sas +++ /dev/null @@ -1,263 +0,0 @@ -/*** HELP START ***//** - @file - @brief Takes a table of running jobs and waits for ANY/ALL of them to complete - @details Will poll `/jobs/{jobId}/state` at set intervals until ANY or ALL - jobs are completed. Completion is determined by reference to the returned - _state_, as per the following table: - - | state | Wait? | Notes| - |-----------|-------|------| - | idle | yes | We assume processing will continue. Beware of idle sessions with no code submitted! | - | pending | yes | Job is preparing to run | - | running | yes | Job is running| - | canceled | no | Job was cancelled| - | completed | no | Job finished - does not mean it was successful. Check stateDetails| - | failed | no | Job failed to execute, could be a problem when calling the apis| - - - ## Example - - First, compile the macros: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - Next, create a job (in this case, as a web service): - - filename ft15f001 temp; - parmcards4; - data ; - rand=ranuni(0)*1000000; - do x=1 to rand; - y=rand*x; - output; - end; - run; - ;;;; - %mv_createwebservice(path=/Public/temp,name=demo) - - Then, execute the job,multiple times, and wait for them all to finish: - - %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds1) - %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds2) - %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds3) - %mv_jobexecute(path=/Public/temp,name=demo,outds=work.ds4) - - data work.jobs; - set work.ds1 work.ds2 work.ds3 work.ds4; - where method='GET' and rel='state'; - run; - - %mv_jobwaitfor(ALL,inds=work.jobs,outds=work.jobstates) - - Delete the job: - - %mv_deletejes(path=/Public/temp,name=demo) - - @param [in] access_token_var= The global macro variable to contain the access - token - @param [in] grant_type= valid values: - - - password - - authorization_code - - detect - will check if access_token exists, if not will use sas_services - if a SASStudioV session else authorization_code. Default option. - - sas_services - will use oauth_bearer=sas_services - - @param [in] action=Either ALL (to wait for every job) or ANY (if one job - completes, processing will continue). Default=ALL. - @param [in] inds= The input dataset containing the list of job uris, in the - following format: `/jobExecution/jobs/&JOBID./state` and the corresponding - job name. The uri should be in a `uri` variable, and the job path/name - should be in a `_program` variable. - @param [in] raise_err=0 Set to 1 to raise SYSCC when a job does not complete - succcessfully - @param [in] mdebug= set to 1 to enable DEBUG messages - @param [out] outds= The output dataset containing the list of states by job - (default=work.mv_jobexecute) - @param [out] outref= A fileref to which the spawned job logs should be - appended. - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

Dependencies

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_existvar.sas - @li mf_nobs.sas - @li mv_getjoblog.sas - -**//*** HELP END ***/ - -%macro mv_jobwaitfor(action - ,access_token_var=ACCESS_TOKEN - ,grant_type=sas_services - ,inds=0 - ,outds=work.mv_jobwaitfor - ,outref=0 - ,raise_err=0 - ,mdebug=0 - ); -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%local oauth_bearer; -%if &grant_type=detect %then %do; - %if %symexist(&access_token_var) %then %let grant_type=authorization_code; - %else %let grant_type=sas_services; -%end; -%if &grant_type=sas_services %then %do; - %let oauth_bearer=oauth_bearer=sas_services; - %let &access_token_var=; -%end; - -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password - and &grant_type ne sas_services - ) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort(iftrue=("&inds"="0") - ,mac=&sysmacroname - ,msg=%str(input dataset not provided) -) -%mp_abort(iftrue=(%mf_existvar(&inds,uri)=0) - ,mac=&sysmacroname - ,msg=%str(The URI variable was not found in the input dataset(&inds)) -) -%mp_abort(iftrue=(%mf_existvar(&inds,_program)=0) - ,mac=&sysmacroname - ,msg=%str(The _PROGRAM variable was not found in the input dataset(&inds)) -) - -%if %mf_nobs(&inds)=0 %then %do; - %put NOTE: Zero observations in &inds, &sysmacroname will now exit; - %return; -%end; - -options noquotelenmax; -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -data _null_; - length jobparams $32767; - set &inds end=last; - call symputx(cats('joburi',_n_),substr(uri,1,55),'l'); - call symputx(cats('jobname',_n_),_program,'l'); - call symputx(cats('jobparams',_n_),jobparams,'l'); - if last then call symputx('uricnt',_n_,'l'); -run; - -%local runcnt; -%if &action=ALL %then %let runcnt=&uricnt; -%else %if &action=ANY %then %let runcnt=1; -%else %let runcnt=&uricnt; - -%local fname0 ; -%let fname0=%mf_getuniquefileref(); - -data &outds; - format _program uri $128. state $32. stateDetails $32. timestamp datetime19. - jobparams $32767.; - call missing (of _all_); - stop; -run; - -%local i; -%do i=1 %to &uricnt; - %if "&&joburi&i" ne "0" %then %do; - proc http method='GET' out=&fname0 &oauth_bearer url="&base_uri/&&joburi&i"; - headers "Accept"="application/json" - %if &grant_type=authorization_code %then %do; - "Authorization"="Bearer &&&access_token_var" - %end; ; - run; - %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 - %then %do; - data _null_;infile &fname0;input;putlog _infile_;run; - %mp_abort(mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) - ) - %end; - - %let status=notset; - - %local libref1; - %let libref1=%mf_getuniquelibref(); - libname &libref1 json fileref=&fname0; - - data _null_; - length state stateDetails $32; - set &libref1..root; - call symputx('status',state,'l'); - call symputx('stateDetails',stateDetails,'l'); - run; - - libname &libref1 clear; - - %if &status=completed or &status=failed or &status=canceled %then %do; - %local plainuri; - %let plainuri=%substr(&&joburi&i,1,55); - proc sql; - insert into &outds set - _program="&&jobname&i", - uri="&plainuri", - state="&status", - stateDetails=symget("stateDetails"), - timestamp=datetime(), - jobparams=symget("jobparams&i"); - %let joburi&i=0; /* do not re-check */ - /* fetch log */ - %if %str(&outref) ne 0 %then %do; - %mv_getjoblog(uri=&plainuri,outref=&outref,mdebug=&mdebug) - %end; - %end; - %else %if &status=idle or &status=pending or &status=running %then %do; - data _null_; - call sleep(1,1); - run; - %end; - %else %do; - %mp_abort(mac=&sysmacroname - ,msg=%str(status &status not expected!!) - ) - %end; - - %if (&raise_err) %then %do; - %if (&status = canceled or &status = failed or %length(&stateDetails)>0) - %then %do; - %if ("&stateDetails" = "%str(war)ning") %then %let SYSCC=4; - %else %let SYSCC=5; - %put %str(ERR)OR: Job &&jobname&i. did not complete. &stateDetails; - %return; - %end; - %end; - - %end; - %if &i=&uricnt %then %do; - %local goback; - %let goback=0; - proc sql noprint; - select count(*) into:goback from &outds; - %if &goback lt &runcnt %then %let i=0; - %end; -%end; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%else %do; - /* clear refs */ - filename &fname0 clear; -%end; -%mend mv_jobwaitfor; diff --git a/003_macros/mv_registerclient.sas b/003_macros/mv_registerclient.sas deleted file mode 100644 index 36c7f57..0000000 --- a/003_macros/mv_registerclient.sas +++ /dev/null @@ -1,287 +0,0 @@ -/*** HELP START ***//** - @file mv_registerclient.sas - @brief Register Client and Secret (admin task) - @details When building apps on SAS Viya, a client id and secret are usually - required. In order to generate them, the Consul Token is required. To access - this token, you need to be a system administrator (it is not enough to be in - the SASAdministrator group in SAS Environment Manager). - - If you are registering a lot of clients / secrets, you may find it more - convenient to use the [Viya Token Generator] - (https://sasjs.io/apps/#viya-client-token-generator) (a SASjs Web App to - automate the generation of clients & secrets with various settings). - - For further information on clients / secrets, see; - @li https://developer.sas.com/reference/auth/#register - @li https://proc-x.com/2019/01/authentication-to-sas-viya-a-couple-of-approaches - @li https://cli.sasjs.io/faq/#how-can-i-obtain-a-viya-client-and-secret - - The default viyaroot location is: `/opt/sas/viya/config` - - Usage: - - %* compile macros; - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* generate random client details with openid scope; - %mv_registerclient(scopes=openid ) - - %* generate random client using consul token as input parameter; - %mv_registerclient(consul_token=12x34sa43v2345n234lasd) - - %* specific client with just openid scope; - %mv_registerclient(client_id=YourClient - ,client_secret=YourSecret - ,scopes=openid - ) - - %* generate random client with 90/180 second access/refresh token expiry; - %mv_registerclient(scopes=openid - ,access_token_validity=90 - ,refresh_token_validity=180 - ) - - @param [in,out] client_id= The client name. Auto generated if blank. - @param [in,out] client_secret= Client secret. Auto generated if client is - blank. - @param [in] consul_token= (0) Provide the actual consul token value here if - using Viya 4 or above. - @param [in] scopes= (openid) List of space-seperated unquoted scopes - @param [in] grant_type= (authorization_code|refresh_token) Valid values are - "password" or "authorization_code" (unquoted). Pipe seperated. - @param [out] outds=(mv_registerclient) The dataset to contain the registered - client id and secret - @param [in] access_token_validity= (DEFAULT) The access token duration in - seconds. A value of DEFAULT will omit the entry (and use system default) - @param [in] refresh_token_validity= (DEFAULT) The duration of validity of the - refresh token in seconds. A value of DEFAULT will omit the entry (and use - system default) - @param [in] client_name= (DEFAULT) An optional, human readable name for the - client. - @param [in] required_user_groups= A list of group names. If a user does not - belong to all the required groups, the user will not be authenticated and no - tokens are issued to this client for that user. If this field is not - specified, authentication and token issuance proceeds normally. - @param [in] autoapprove= During the auth step the user can choose which scope - to apply. Setting this to true will autoapprove all the client scopes. - @param [in] use_session= If true, access tokens issued to this client will be - associated with an HTTP session and revoked upon logout or time-out. - @param [out] outjson= (_null_) A dataset containing the lines of JSON - submitted. Useful for debugging. - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_loc.sas - @li mf_getquotedstr.sas - @li mf_getuser.sas - @li mp_abort.sas - -**//*** HELP END ***/ - -%macro mv_registerclient(client_id= - ,client_secret= - ,consul_token=0 - ,client_name=DEFAULT - ,scopes=openid - ,grant_type=authorization_code|refresh_token - ,required_user_groups= - ,autoapprove= - ,use_session= - ,outds=mv_registerclient - ,access_token_validity=DEFAULT - ,refresh_token_validity=DEFAULT - ,outjson=_null_ - ); -%local fname1 fname2 fname3 libref access_token url tokloc msg; - -%if client_name=DEFAULT %then %let client_name= - Generated by %mf_getuser() (&sysuserid) on %sysfunc(datetime(),datetime19. - ) using SASjs; - -options noquotelenmax; - -%if "&consul_token"="0" %then %do; - /* first, get consul token needed to get client id / secret */ - %let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default; - %let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token; - - %if %sysfunc(fileexist(&tokloc))=0 %then %do; - %let msg=Unable to access the consul token at &tokloc; - %put &sysmacroname: &msg; - %put Try passing the value in the consul= macro parameter; - %put See docs: https://core.sasjs.io/mv__registerclient_8sas.html; - %mp_abort(mac=mv_registerclient,msg=%str(&msg)) - %end; - - data _null_; - infile "&tokloc"; - input token:$64.; - call symputx('consul_token',token); - run; - - %if "&consul_token"="0" %then %do; - %put &sysmacroname: Unable to source the consul token from &tokloc; - %put It seems your account (&sysuserid) does not have admin rights; - %put Please speak with your platform adminstrator; - %put Docs: https://core.sasjs.io/mv__registerclient_8sas.html; - %abort; - %end; -%end; - -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -/* request the client details */ -%let fname1=%mf_getuniquefileref(); -proc http method='POST' out=&fname1 - url="&base_uri/SASLogon/oauth/clients/consul?callback=false%str(&)%trim( - )serviceId=app"; - headers "X-Consul-Token"="&consul_token"; -run; - -%put &=SYS_PROCHTTP_STATUS_CODE; -%put &=SYS_PROCHTTP_STATUS_PHRASE; - -%let libref=%mf_getuniquelibref(); -libname &libref JSON fileref=&fname1; - -/* extract the token */ -data _null_; - set &libref..root; - call symputx('access_token',access_token,'l'); -run; - -/** - * register the new client - */ -%let fname2=%mf_getuniquefileref(); -%if x&client_id.x=xx %then %do; - %let client_id=client_%sysfunc(ranuni(0),hex16.); - %let client_secret=secret_%sysfunc(ranuni(0),hex16.); -%end; - -%let scopes=%sysfunc(coalescec(&scopes,openid)); -%let scopes=%mf_getquotedstr(&scopes,QUOTE=D,indlm=|); -%let grant_type=%mf_getquotedstr(&grant_type,QUOTE=D,indlm=|); -%let required_user_groups= - %mf_getquotedstr(&required_user_groups,QUOTE=D,indlm=|); - -data _null_; - file &fname2; - length clientid clientsecret clientname scope grant_types reqd_groups - autoapprove $256.; - clientid='"client_id":'!!quote(trim(symget('client_id'))); - clientsecret=',"client_secret":'!!quote(trim(symget('client_secret'))); - clientname=',"name":'!!quote(trim(symget('client_name'))); - scope=',"scope":['!!symget('scopes')!!']'; - grant_types=symget('grant_type'); - if grant_types = '""' then grant_types =''; - grant_types=cats(',"authorized_grant_types": [',grant_types,']'); - reqd_groups=symget('required_user_groups'); - if reqd_groups = '""' then reqd_groups =''; - else reqd_groups=cats(',"required_user_groups":[',reqd_groups,']'); - autoapprove=trim(symget('autoapprove')); - if not missing(autoapprove) then autoapprove= - cats(',"autoapprove":',autoapprove); - use_session=trim(symget('use_session')); - if not missing(use_session) then use_session= - cats(',"use_session":',use_session); - - put '{' clientid ; - put clientsecret ; - put clientname; - put scope; - put grant_types; - if not missing(reqd_groups) then put reqd_groups; - put autoapprove; - put use_session; -%if &access_token_validity ne DEFAULT %then %do; - put ',"access_token_validity":' "&access_token_validity"; -%end; -%if &refresh_token_validity ne DEFAULT %then %do; - put ',"refresh_token_validity":' "&refresh_token_validity"; -%end; - - put ',"redirect_uri": "urn:ietf:wg:oauth:2.0:oob"'; - put '}'; -run; - -%let fname3=%mf_getuniquefileref(); -proc http method='POST' in=&fname2 out=&fname3 - url="&base_uri/SASLogon/oauth/clients"; - headers "Content-Type"="application/json" - "Authorization"="Bearer &access_token"; -run; - -/* show response */ -%local err; -%let err=NONE; -data _null_; - infile &fname3; - input; - if _infile_=:'{"err'!!'or":' then do; - length message $32767; - message=scan(_infile_,-2,'"'); - call symputx('err',message,'l'); - end; -run; -%if "&err" ne "NONE" %then %do; - %put %str(ERR)OR: &err; -%end; - -/* prepare url */ -%if %index(%superq(grant_type),authorization_code) %then %do; - data _null_; - if symexist('_baseurl') then do; - url=symget('_baseurl'); - if subpad(url,length(url)-9,9)='SASStudio' - then url=substr(url,1,length(url)-11); - else url="&systcpiphostname"; - end; - else url="&systcpiphostname"; - call symputx('url',url); - run; -%end; - -%put Please provide the following details to the developer:; -%put ; -%put CLIENT_ID=&client_id; -%put CLIENT_SECRET=&client_secret; -%put GRANT_TYPE=&grant_type; -%put; -%if %index(%superq(grant_type),authorization_code) %then %do; - /* cannot use base_uri here as it includes the protocol which may be incorrect - externally */ - %put NOTE: Visit the link below and select 'openid' to get the grant code:; - %put NOTE- ; - %put NOTE- &url/SASLogon/oauth/authorize?client_id=&client_id%str(&)%trim( - )response_type=code; - %put NOTE- ; -%end; - -data &outds; - client_id=symget('client_id'); - client_secret=symget('client_secret'); - error=symget('err'); -run; - -data &outjson; - infile &fname2; - input; - line=_infile_; -run; - -/* clear refs */ -filename &fname1 clear; -filename &fname2 clear; -filename &fname3 clear; -libname &libref clear; - -%mend mv_registerclient; diff --git a/003_macros/mv_tokenauth.sas b/003_macros/mv_tokenauth.sas deleted file mode 100644 index 7714352..0000000 --- a/003_macros/mv_tokenauth.sas +++ /dev/null @@ -1,149 +0,0 @@ -/*** HELP START ***//** - @file mv_tokenauth.sas - @brief Get initial Refresh and Access Tokens - @details Before a Refresh Token can be obtained, the client must be - registered by an administrator. This can be done using the - `mv_registerclient` macro, after which the user must visit a URL to get an - additional code (if using oauth). - - That code (or username / password) is used here to get the Refresh Token - (and an initial Access Token). THIS MACRO CAN ONLY BE USED ONCE - further - access tokens can be obtained using the `mv_gettokenrefresh` macro. - - Access tokens expire frequently (every 10 hours or so) whilst refresh tokens - expire periodically (every month or so). This is all configurable. - - Usage: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - - %mv_registerclient(outds=clientinfo) - - %mv_tokenauth(inds=clientinfo,code=LD39EpalOf) - - A great article for explaining all these steps is available here: - - https://blogs.sas.com/content/sgf/2019/01/25/authentication-to-sas-viya/ - - @param [in] inds= A dataset containing client_id, client_secret, and auth_code - @param [in] outds= A dataset containing access_token and refresh_token - @param [in] client_id= The client name - @param [in] client_secret= client secret - @param [in] grant_type= valid values are "password" or "authorization_code" - (unquoted). The default is authorization_code. - @param [in] code= () - If grant_type=authorization_code then provide the necessary code here - @param [in] user= If grant_type=password then provide the username here - @param [in] pass= If grant_type=password then provide the password here - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] refresh_token_var= (REFRESH_TOKEN) - The global macro variable to contain the refresh token - @param [in] base_uri= The Viya API server location - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_existds.sas - -**//*** HELP END ***/ - -%macro mv_tokenauth(inds=mv_registerclient - ,outds=mv_tokenauth - ,client_id=someclient - ,client_secret=somesecret - ,grant_type=authorization_code - ,code= - ,user= - ,pass= - ,access_token_var=ACCESS_TOKEN - ,refresh_token_var=REFRESH_TOKEN - ,base_uri=#NOTSET# - ); -%global &access_token_var &refresh_token_var; - -%local fref1 fref2 libref; - -/* test the validity of inputs */ -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%if %mf_existds(&inds) %then %do; - data _null_; - set &inds; - call symputx('client_id',client_id,'l'); - call symputx('client_secret',client_secret,'l'); - if not missing(auth_code) then call symputx('code',auth_code,'l'); - run; -%end; - -%mp_abort(iftrue=(&grant_type=authorization_code and %str(&code)=%str()) - ,mac=&sysmacroname - ,msg=%str(Authorization code required) -) - -%mp_abort(iftrue=( - &grant_type=password and (%str(&user)=%str() or %str(&pass)=%str())) - ,mac=&sysmacroname - ,msg=%str(username / password required) -) - -/* prepare appropriate grant type */ -%let fref1=%mf_getuniquefileref(); - -data _null_; - file &fref1; - if "&grant_type"='authorization_code' then string=cats( - 'grant_type=authorization_code&code=',symget('code')); - else string=cats('grant_type=password&username=',symget('user') - ,'&password=',symget(pass)); - call symputx('grantstring',cats("'",string,"'")); -run; -/*data _null_;infile &fref1;input;put _infile_;run;*/ - -/** - * Request access token - */ -%if &base_uri=#NOTSET# %then %let base_uri=%mf_getplatform(VIYARESTAPI); - -%let fref2=%mf_getuniquefileref(); -proc http method='POST' in=&grantstring out=&fref2 - url="&base_uri/SASLogon/oauth/token" - WEBUSERNAME="&client_id" - WEBPASSWORD="&client_secret" - AUTH_BASIC; - headers "Accept"="application/json" - "Content-Type"="application/x-www-form-urlencoded"; -run; -/*data _null_;infile &fref2;input;put _infile_;run;*/ - -/** - * Extract access / refresh tokens - */ - -%let libref=%mf_getuniquelibref(); -libname &libref JSON fileref=&fref2; - -/* extract the tokens */ -data &outds; - set &libref..root; - call symputx("&access_token_var",access_token); - call symputx("&refresh_token_var",refresh_token); -run; - - -libname &libref clear; -filename &fref1 clear; -filename &fref2 clear; - -%mend mv_tokenauth; diff --git a/003_macros/mv_tokenrefresh.sas b/003_macros/mv_tokenrefresh.sas deleted file mode 100644 index b655713..0000000 --- a/003_macros/mv_tokenrefresh.sas +++ /dev/null @@ -1,134 +0,0 @@ -/*** HELP START ***//** - @file mv_tokenrefresh.sas - @brief Get an additional access token using a refresh token - @details Before an access token can be obtained, a refresh token is required - For that, check out the `mv_tokenauth` macro. - - Usage: - - * prep work - register client, get refresh token, save it for later use ; - %mv_registerclient(outds=client) - %mv_tokenauth(inds=client,code=wKDZYTEPK6) - data _null_; - file "~/refresh.token"; - put "&refresh_token"; - run; - - * now do the things n stuff; - data _null_; - infile "~/refresh.token"; - input; - call symputx('refresh_token',_infile_); - run; - %mv_tokenrefresh(client_id=&client - ,client_secret=&secret - ) - - A great article for explaining all these steps is available here: - - https://blogs.sas.com/content/sgf/2019/01/25/authentication-to-sas-viya/ - - @param [in] inds= A dataset containing client_id and client_secret - @param [in] outds= A dataset containing access_token and refresh_token - @param [in] client_id= The client name (alternative to inds) - @param [in] client_secret= client secret (alternative to inds) - @param [in] grant_type= valid values are "password" or "authorization_code" - (unquoted). The default is authorization_code. - @param [in] user= If grant_type=password then provide the username here - @param [in] pass= If grant_type=password then provide the password here - @param [in] access_token_var= (ACCESS_TOKEN) - The global macro variable to contain the access token - @param [in] refresh_token_var= (REFRESH_TOKEN) - The global macro variable containing the refresh token - - @version VIYA V.03.04 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mf_existds.sas - -**//*** HELP END ***/ - -%macro mv_tokenrefresh(inds=mv_registerclient - ,outds=mv_tokenrefresh - ,client_id=someclient - ,client_secret=somesecret - ,grant_type=authorization_code - ,user= - ,pass= - ,access_token_var=ACCESS_TOKEN - ,refresh_token_var=REFRESH_TOKEN - ); -%global &access_token_var &refresh_token_var; -options noquotelenmax; - -%local fref1 libref; - -/* test the validity of inputs */ -%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password) - ,mac=&sysmacroname - ,msg=%str(Invalid value for grant_type: &grant_type) -) - -%mp_abort( - iftrue=(&grant_type=password and (%str(&user)=%str() or %str(&pass)=%str())) - ,mac=&sysmacroname - ,msg=%str(username / password required) -) - -%if %mf_existds(&inds) %then %do; - data _null_; - set &inds; - call symputx('client_id',client_id,'l'); - call symputx('client_secret',client_secret,'l'); - call symputx("&refresh_token_var",&refresh_token_var,'l'); - run; -%end; - -%mp_abort(iftrue=(%str(&client_id)=%str() or %str(&client_secret)=%str()) - ,mac=&sysmacroname - ,msg=%str(client / secret must both be provided) -) - -/** - * Request access token - */ -%local base_uri; /* location of rest apis */ -%let base_uri=%mf_getplatform(VIYARESTAPI); - -%let fref1=%mf_getuniquefileref(); -proc http method='POST' - in="grant_type=refresh_token%nrstr(&)refresh_token=&&&refresh_token_var" - out=&fref1 - url="&base_uri/SASLogon/oauth/token" - WEBUSERNAME="&client_id" - WEBPASSWORD="&client_secret" - AUTH_BASIC; - headers "Accept"="application/json" - "Content-Type"="application/x-www-form-urlencoded"; -run; -/*data _null_;infile &fref1;input;put _infile_;run;*/ - -/** - * Extract access / refresh tokens - */ - -%let libref=%mf_getuniquelibref(); -libname &libref JSON fileref=&fref1; - -/* extract the token */ -data &outds; - set &libref..root; - call symputx("&access_token_var",access_token); - call symputx("&refresh_token_var",refresh_token); -run; - - -libname &libref clear; -filename &fref1 clear; - -%mend mv_tokenrefresh; diff --git a/003_macros/mv_webout.sas b/003_macros/mv_webout.sas deleted file mode 100644 index 71e27d0..0000000 --- a/003_macros/mv_webout.sas +++ /dev/null @@ -1,271 +0,0 @@ -/*** HELP START ***//** - @file - @brief Send data to/from the SAS Viya Job Execution Service - @details This macro should be added to the start of each Job Execution - Service, **immediately** followed by a call to: - - %mv_webout(FETCH) - - This will read all the input data and create same-named SAS datasets in the - WORK library. You can then insert your code, and send data back using the - following syntax: - - data some datasets; * make some data ; - retain some columns; - run; - - %mv_webout(OPEN) - %mv_webout(ARR,some) * Array format, fast, suitable for large tables ; - %mv_webout(OBJ,datasets) * Object format, easier to work with ; - %mv_webout(CLOSE) - - - @param [in] action Either OPEN, ARR, OBJ or CLOSE - @param [in] ds The dataset to send back to the frontend - @param [in] _webout= fileref for returning the json - @param [out] fref=(_mvwtemp) Temp fileref to which to write the output - @param [out] dslabel= value to use instead of table name for sending to JSON - @param [in] fmt= (N) Setting Y converts all vars to their formatted values - @param [in] stream=(Y) Change to N if not streaming to _webout - @param [in] missing= (NULL) Special numeric missing values can be sent as NULL - (eg `null`) or as STRING values (eg `".a"` or `".b"`) - @param [in] showmeta= (N) Set to Y to output metadata alongside each table, - such as the column formats and types. The metadata is contained inside an - object with the same name as the table but prefixed with a dollar sign - ie, - `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` - @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows - that should be converted to output JSON - @param [in] workobs= (0) When set to a positive integer, will create a new - output object (WORK) which contains this number of observations from all - tables in the WORK library. - -

SAS Macros

- @li mp_jsonout.sas - @li mf_getuser.sas - -

Related Macros

- @li ms_webout.sas - @li mm_webout.sas - - @version Viya 3.3 - @author Allan Bowe, source: https://github.com/sasjs/core - -**//*** HELP END ***/ -%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=N,stream=Y,missing=NULL - ,showmeta=N,maxobs=MAX,workobs=0 -); -%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name - sasjs_tables SYS_JES_JOB_URI; -%if %index("&_debug",log) %then %let _debug=131; - -%local i tempds table; -%let action=%upcase(&action); - -%if &action=FETCH %then %do; - %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 131 %then %do; - options mprint notes mprintnest; - %end; - - %if not %symexist(_webin_fileuri1) %then %do; - %let _webin_file_count=%eval(&_webin_file_count+0); - %let _webin_fileuri1=&_webin_fileuri; - %let _webin_name1=&_webin_name; - %end; - - /* if the sasjs_tables param is passed, we expect param based upload */ - %if %length(&sasjs_tables.X)>1 %then %do; - - /* convert data from macro variables to datasets */ - %do i=1 %to %sysfunc(countw(&sasjs_tables)); - %let table=%scan(&sasjs_tables,&i,%str( )); - %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; - data _null_; - file "%sysfunc(pathname(work))/&table..csv" recfm=n; - retain nrflg 0; - length line $32767; - do i=1 to &&sasjs&i.data0; - if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); - else line=symget(cats("sasjs&i.data",i)); - if i=1 and substr(line,1,7)='%nrstr(' then do; - nrflg=1; - line=substr(line,8); - end; - if i=&&sasjs&i.data0 and nrflg=1 then do; - line=substr(line,1,length(line)-1); - end; - put line +(-1) @; - end; - run; - data _null_; - infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; - input; - if _n_=1 then call symputx('input_statement',_infile_); - list; - data work.&table; - infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd - termstr=crlf; - input &input_statement; - run; - %end; - %end; - %else %do i=1 %to &_webin_file_count; - /* read in any files that are sent */ - /* this part needs refactoring for wide files */ - filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999; - data _null_; - infile indata termstr=crlf lrecl=32767; - input; - if _n_=1 then call symputx('input_statement',_infile_); - %if %str(&_debug) ge 131 %then %do; - if _n_<20 then putlog _infile_; - else stop; - %end; - %else %do; - stop; - %end; - run; - data &&_webin_name&i; - infile indata firstobs=2 dsd termstr=crlf ; - input &input_statement; - run; - %let sasjs_tables=&sasjs_tables &&_webin_name&i; - %end; -%end; -%else %if &action=OPEN %then %do; - /* setup webout */ - OPTIONS NOBOMFILE; - %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; - filename _webout temp lrecl=999999 mod; - %end; - %else %do; - filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" - name="_webout.json" lrecl=999999 mod; - %end; - - /* setup temp ref */ - %if %upcase(&fref) ne _WEBOUT %then %do; - filename &fref temp lrecl=999999 permission='A::u::rwx,A::g::rw-,A::o::---'; - %end; - - /* setup json */ - data _null_;file &fref; - put '{"SYSDATE" : "' "&SYSDATE" '"'; - put ',"SYSTIME" : "' "&SYSTIME" '"'; - run; -%end; -%else %if &action=ARR or &action=OBJ %then %do; - %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref - ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs - ) -%end; -%else %if &action=CLOSE %then %do; - %if %str(&workobs) > 0 %then %do; - /* send back first XX records of each work table for debugging */ - data;run;%let tempds=%scan(&syslast,2,.); - ods output Members=&tempds; - proc datasets library=WORK memtype=data; - %local wtcnt;%let wtcnt=0; - data _null_; - set &tempds; - if not (upcase(name) =:"DATA"); /* ignore temp datasets */ - i+1; - call symputx(cats('wt',i),name,'l'); - call symputx('wtcnt',i,'l'); - data _null_; file &fref mod; put ",""WORK"":{"; - %do i=1 %to &wtcnt; - %let wt=&&wt&i; - data _null_; file &fref mod; - dsid=open("WORK.&wt",'is'); - nlobs=attrn(dsid,'NLOBS'); - nvars=attrn(dsid,'NVARS'); - rc=close(dsid); - if &i>1 then put ','@; - put " ""&wt"" : {"; - put '"nlobs":' nlobs; - put ',"nvars":' nvars; - %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y - ,maxobs=&workobs - ) - data _null_; file &fref mod;put "}"; - %end; - data _null_; file &fref mod;put "}";run; - %end; - - /* close off json */ - data _null_;file &fref mod; - length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; - put ",""_DEBUG"" : ""&_debug"" "; - _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); - put ',"_PROGRAM" : ' _PROGRAM ; - autoexec=quote(urlencode(trim(getoption('autoexec')))); - put ',"AUTOEXEC" : ' autoexec; - put ",""MF_GETUSER"" : ""%mf_getuser()"" "; - SYS_JES_JOB_URI=quote(trim(resolve(symget('SYS_JES_JOB_URI')))); - put ',"SYS_JES_JOB_URI" : ' SYS_JES_JOB_URI ; - put ",""SYSJOBID"" : ""&sysjobid"" "; - put ",""SYSCC"" : ""&syscc"" "; - syserrortext=cats(symget('syserrortext')); - if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syserrortext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syserrortext) - )))))))))))))!!'"'; - end; - else syserrortext=cats('"',syserrortext,'"'); - put ',"SYSERRORTEXT" : ' syserrortext; - put ",""SYSHOSTNAME"" : ""&syshostname"" "; - put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; - put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; - SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); - put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; - put ",""SYSJOBID"" : ""&sysjobid"" "; - put ",""SYSSCPL"" : ""&sysscpl"" "; - put ",""SYSSITE"" : ""&syssite"" "; - put ",""SYSUSERID"" : ""&sysuserid"" "; - sysvlong=quote(trim(symget('sysvlong'))); - put ',"SYSVLONG" : ' sysvlong; - syswarningtext=cats(symget('syswarningtext')); - if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syswarningtext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syswarningtext) - )))))))))))))!!'"'; - end; - else syswarningtext=cats('"',syswarningtext,'"'); - put ',"SYSWARNINGTEXT" : ' syswarningtext; - put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" '; - length memsize $32; - memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; - memsize=quote(cats(memsize)); - put ',"MEMSIZE" : ' memsize; - put "}"; - - %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; - data _null_; rc=fcopy("&fref","_webout");run; - %end; - -%end; - -%mend mv_webout; diff --git a/002_macros/ml_gsubfile.sas b/004_macros/ml_gsubfile.sas similarity index 100% rename from 002_macros/ml_gsubfile.sas rename to 004_macros/ml_gsubfile.sas diff --git a/002_macros/ml_json.sas b/004_macros/ml_json.sas similarity index 100% rename from 002_macros/ml_json.sas rename to 004_macros/ml_json.sas diff --git a/005_macros/mf_isblank.sas b/005_macros/mf_isblank.sas deleted file mode 100644 index 7181689..0000000 --- a/005_macros/mf_isblank.sas +++ /dev/null @@ -1,27 +0,0 @@ -/*** HELP START ***//** - @file mf_isblank.sas - @brief Checks whether a macro variable is empty (blank) - @details Simply performs: - - %sysevalf(%superq(param)=,boolean) - - Usage: - - %put mf_isblank(&var); - - inspiration: - https://support.sas.com/resources/papers/proceedings09/022-2009.pdf - - @param [in] Param VALUE to be checked - - @return output returns 1 (if blank) else 0 - - @version 9.2 -**//*** HELP END ***/ - -%macro mf_isblank(param -)/*/STORE SOURCE*/; - - %sysevalf(%superq(param)=,boolean) - -%mend mf_isblank; diff --git a/005_macros/mfs_httpheader.sas b/005_macros/mfs_httpheader.sas deleted file mode 100644 index 5378da8..0000000 --- a/005_macros/mfs_httpheader.sas +++ /dev/null @@ -1,53 +0,0 @@ -/*** HELP START ***//** - @file - @brief Sets the http headers in the SASjs/server response - @details For GET requests, SASjs server will use the file generated by this - macro for setting the appropriate http headers in the response. - - It works by writing a file to the session directory, that is then ingested by - the server. - - The location of this file is driven by the global variable - `sasjs_stpsrv_header_loc` which is made available in the autoexec. - - Usage: - - %mfs_httpheader(Content-Type,application/csv) - - @param [in] header_name Name of the http header to set - @param [in] header_value Value of the http header to set - -

Related Macros

- @li mcf_stpsrv_header.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mfs_httpheader(header_name - ,header_value -)/*/STORE SOURCE*/; -%global sasjs_stpsrv_header_loc; -%local fref fid i; - -%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do; - %put &=fref &=sasjs_stpsrv_header_loc; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%let fid=%sysfunc(fopen(&fref,A)); - -%if &fid=0 %then %do; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value))); -%let rc=%sysfunc(fwrite(&fid)); - -%let rc=%sysfunc(fclose(&fid)); -%let rc=%sysfunc(filename(&fref)); - -%mend mfs_httpheader; diff --git a/002_macros/mm_adduser2group.sas b/005_macros/mm_adduser2group.sas similarity index 100% rename from 002_macros/mm_adduser2group.sas rename to 005_macros/mm_adduser2group.sas diff --git a/003_macros/mm_assigndirectlib.sas b/005_macros/mm_assigndirectlib.sas similarity index 100% rename from 003_macros/mm_assigndirectlib.sas rename to 005_macros/mm_assigndirectlib.sas diff --git a/002_macros/mm_assignlib.sas b/005_macros/mm_assignlib.sas similarity index 100% rename from 002_macros/mm_assignlib.sas rename to 005_macros/mm_assignlib.sas diff --git a/002_macros/mm_createapplication.sas b/005_macros/mm_createapplication.sas similarity index 100% rename from 002_macros/mm_createapplication.sas rename to 005_macros/mm_createapplication.sas diff --git a/002_macros/mm_createdataset.sas b/005_macros/mm_createdataset.sas similarity index 100% rename from 002_macros/mm_createdataset.sas rename to 005_macros/mm_createdataset.sas diff --git a/002_macros/mm_createdocument.sas b/005_macros/mm_createdocument.sas similarity index 100% rename from 002_macros/mm_createdocument.sas rename to 005_macros/mm_createdocument.sas diff --git a/002_macros/mm_createfolder.sas b/005_macros/mm_createfolder.sas similarity index 100% rename from 002_macros/mm_createfolder.sas rename to 005_macros/mm_createfolder.sas diff --git a/002_macros/mm_createlibrary.sas b/005_macros/mm_createlibrary.sas similarity index 100% rename from 002_macros/mm_createlibrary.sas rename to 005_macros/mm_createlibrary.sas diff --git a/003_macros/mm_createstp.sas b/005_macros/mm_createstp.sas similarity index 100% rename from 003_macros/mm_createstp.sas rename to 005_macros/mm_createstp.sas diff --git a/002_macros/mm_createwebservice.sas b/005_macros/mm_createwebservice.sas similarity index 100% rename from 002_macros/mm_createwebservice.sas rename to 005_macros/mm_createwebservice.sas diff --git a/002_macros/mm_deletedocument.sas b/005_macros/mm_deletedocument.sas similarity index 100% rename from 002_macros/mm_deletedocument.sas rename to 005_macros/mm_deletedocument.sas diff --git a/002_macros/mm_deletelibrary.sas b/005_macros/mm_deletelibrary.sas similarity index 100% rename from 002_macros/mm_deletelibrary.sas rename to 005_macros/mm_deletelibrary.sas diff --git a/002_macros/mm_deletestp.sas b/005_macros/mm_deletestp.sas similarity index 100% rename from 002_macros/mm_deletestp.sas rename to 005_macros/mm_deletestp.sas diff --git a/002_macros/mm_getauthinfo.sas b/005_macros/mm_getauthinfo.sas similarity index 100% rename from 002_macros/mm_getauthinfo.sas rename to 005_macros/mm_getauthinfo.sas diff --git a/002_macros/mm_getcols.sas b/005_macros/mm_getcols.sas similarity index 100% rename from 002_macros/mm_getcols.sas rename to 005_macros/mm_getcols.sas diff --git a/002_macros/mm_getdetails.sas b/005_macros/mm_getdetails.sas similarity index 100% rename from 002_macros/mm_getdetails.sas rename to 005_macros/mm_getdetails.sas diff --git a/002_macros/mm_getdirectories.sas b/005_macros/mm_getdirectories.sas similarity index 100% rename from 002_macros/mm_getdirectories.sas rename to 005_macros/mm_getdirectories.sas diff --git a/002_macros/mm_getdocument.sas b/005_macros/mm_getdocument.sas similarity index 100% rename from 002_macros/mm_getdocument.sas rename to 005_macros/mm_getdocument.sas diff --git a/002_macros/mm_getfoldermembers.sas b/005_macros/mm_getfoldermembers.sas similarity index 100% rename from 002_macros/mm_getfoldermembers.sas rename to 005_macros/mm_getfoldermembers.sas diff --git a/002_macros/mm_getfoldertree.sas b/005_macros/mm_getfoldertree.sas similarity index 100% rename from 002_macros/mm_getfoldertree.sas rename to 005_macros/mm_getfoldertree.sas diff --git a/002_macros/mm_getgroupmembers.sas b/005_macros/mm_getgroupmembers.sas similarity index 100% rename from 002_macros/mm_getgroupmembers.sas rename to 005_macros/mm_getgroupmembers.sas diff --git a/003_macros/mm_getgroups.sas b/005_macros/mm_getgroups.sas similarity index 100% rename from 003_macros/mm_getgroups.sas rename to 005_macros/mm_getgroups.sas diff --git a/002_macros/mm_getlibmetadiffs.sas b/005_macros/mm_getlibmetadiffs.sas similarity index 100% rename from 002_macros/mm_getlibmetadiffs.sas rename to 005_macros/mm_getlibmetadiffs.sas diff --git a/002_macros/mm_getlibs.sas b/005_macros/mm_getlibs.sas similarity index 100% rename from 002_macros/mm_getlibs.sas rename to 005_macros/mm_getlibs.sas diff --git a/002_macros/mm_getobjects.sas b/005_macros/mm_getobjects.sas similarity index 100% rename from 002_macros/mm_getobjects.sas rename to 005_macros/mm_getobjects.sas diff --git a/002_macros/mm_getpublictypes.sas b/005_macros/mm_getpublictypes.sas similarity index 100% rename from 002_macros/mm_getpublictypes.sas rename to 005_macros/mm_getpublictypes.sas diff --git a/002_macros/mm_getrepos.sas b/005_macros/mm_getrepos.sas similarity index 100% rename from 002_macros/mm_getrepos.sas rename to 005_macros/mm_getrepos.sas diff --git a/002_macros/mm_getroles.sas b/005_macros/mm_getroles.sas similarity index 100% rename from 002_macros/mm_getroles.sas rename to 005_macros/mm_getroles.sas diff --git a/002_macros/mm_getservercontexts.sas b/005_macros/mm_getservercontexts.sas similarity index 100% rename from 002_macros/mm_getservercontexts.sas rename to 005_macros/mm_getservercontexts.sas diff --git a/002_macros/mm_getstpcode.sas b/005_macros/mm_getstpcode.sas similarity index 100% rename from 002_macros/mm_getstpcode.sas rename to 005_macros/mm_getstpcode.sas diff --git a/002_macros/mm_getstpinfo.sas b/005_macros/mm_getstpinfo.sas similarity index 100% rename from 002_macros/mm_getstpinfo.sas rename to 005_macros/mm_getstpinfo.sas diff --git a/002_macros/mm_getstps.sas b/005_macros/mm_getstps.sas similarity index 100% rename from 002_macros/mm_getstps.sas rename to 005_macros/mm_getstps.sas diff --git a/002_macros/mm_gettableid.sas b/005_macros/mm_gettableid.sas similarity index 100% rename from 002_macros/mm_gettableid.sas rename to 005_macros/mm_gettableid.sas diff --git a/002_macros/mm_gettables.sas b/005_macros/mm_gettables.sas similarity index 100% rename from 002_macros/mm_gettables.sas rename to 005_macros/mm_gettables.sas diff --git a/002_macros/mm_gettree.sas b/005_macros/mm_gettree.sas similarity index 100% rename from 002_macros/mm_gettree.sas rename to 005_macros/mm_gettree.sas diff --git a/002_macros/mm_gettypes.sas b/005_macros/mm_gettypes.sas similarity index 100% rename from 002_macros/mm_gettypes.sas rename to 005_macros/mm_gettypes.sas diff --git a/002_macros/mm_getusers.sas b/005_macros/mm_getusers.sas similarity index 100% rename from 002_macros/mm_getusers.sas rename to 005_macros/mm_getusers.sas diff --git a/002_macros/mm_getwebappsrvprops.sas b/005_macros/mm_getwebappsrvprops.sas similarity index 100% rename from 002_macros/mm_getwebappsrvprops.sas rename to 005_macros/mm_getwebappsrvprops.sas diff --git a/002_macros/mm_spkexport.sas b/005_macros/mm_spkexport.sas similarity index 100% rename from 002_macros/mm_spkexport.sas rename to 005_macros/mm_spkexport.sas diff --git a/002_macros/mm_tree.sas b/005_macros/mm_tree.sas similarity index 100% rename from 002_macros/mm_tree.sas rename to 005_macros/mm_tree.sas diff --git a/002_macros/mm_updateappextension.sas b/005_macros/mm_updateappextension.sas similarity index 100% rename from 002_macros/mm_updateappextension.sas rename to 005_macros/mm_updateappextension.sas diff --git a/002_macros/mm_updatedocument.sas b/005_macros/mm_updatedocument.sas similarity index 100% rename from 002_macros/mm_updatedocument.sas rename to 005_macros/mm_updatedocument.sas diff --git a/002_macros/mm_updatestpservertype.sas b/005_macros/mm_updatestpservertype.sas similarity index 100% rename from 002_macros/mm_updatestpservertype.sas rename to 005_macros/mm_updatestpservertype.sas diff --git a/002_macros/mm_updatestpsourcecode.sas b/005_macros/mm_updatestpsourcecode.sas similarity index 100% rename from 002_macros/mm_updatestpsourcecode.sas rename to 005_macros/mm_updatestpsourcecode.sas diff --git a/002_macros/mm_webout.sas b/005_macros/mm_webout.sas similarity index 100% rename from 002_macros/mm_webout.sas rename to 005_macros/mm_webout.sas diff --git a/005_macros/mp_cntlout.sas b/005_macros/mp_cntlout.sas deleted file mode 100644 index 5326750..0000000 --- a/005_macros/mp_cntlout.sas +++ /dev/null @@ -1,94 +0,0 @@ -/*** HELP START ***//** - @file mp_cntlout.sas - @brief Creates a cntlout dataset in a consistent format - @details The dataset produced by proc format in the cntlout option will vary - according to its contents. - - When dealing with formats from an ETL perspective (eg in [Data Controller for - SAS](https://datacontroller.io)), it is important that the output dataset - has a consistent model (and compariable values). - - This macro makes use of mddl_sas_cntlout.sas to provide the consistent model, - and will left-align the start and end values when dealing with numeric ranges - to enable consistency when checking for differences. - - usage: - - %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport) - - @param [in] libcat The library.catalog reference - @param [in] fmtlist= (0) provide a space separated list of specific formats to - extract - @param [in] iftrue= (1=1) A condition under which the macro should be executed - @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset - -

SAS Macros

- @li mddl_sas_cntlout.sas - @li mf_getuniquename.sas - @li mp_aligndecimal.sas - -

Related Macros

- @li mf_getvarformat.sas - @li mp_getformats.sas - @li mp_loadformat.sas - @li mp_ds2fmtds.sas - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mp_cntlout( - iftrue=(1=1) - ,libcat= - ,cntlout=work.fmtextract - ,fmtlist=0 -)/*/STORE SOURCE*/; -%local ddlds cntlds i; - -%if not(%eval(%unquote(&iftrue))) %then %return; - -%let ddlds=%mf_getuniquename(); -%let cntlds=%mf_getuniquename(); - -%mddl_sas_cntlout(libds=&ddlds) - -%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do; - %let libcat=%scan(&libcat,1,-); -%end; - -proc format lib=&libcat cntlout=&cntlds; -%if "&fmtlist" ne "0" and "&fmtlist" ne "" %then %do; - select - %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); - %scan(&fmtlist,&i,%str( )) - %end; - ; -%end; -run; - -data &cntlout/nonote2err; - if 0 then set &ddlds; - set &cntlds; - by type fmtname notsorted; - - /* align the numeric values to avoid overlapping ranges */ - if type in ("I","N") then do; - %mp_aligndecimal(start,width=16) - %mp_aligndecimal(end,width=16) - end; - - /* create row marker. Data cannot be sorted without it! */ - if first.fmtname then fmtrow=0; - fmtrow+1; - -run; -proc sort; - by type fmtname fmtrow; -run; - -proc sql; -drop table &ddlds,&cntlds; - -%mend mp_cntlout; -/** @endcond */ diff --git a/005_macros/mp_dirlist.sas b/005_macros/mp_dirlist.sas deleted file mode 100644 index aef1e9a..0000000 --- a/005_macros/mp_dirlist.sas +++ /dev/null @@ -1,262 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns all files and subdirectories within a specified parent - @details When used with getattrs=NO, is not OS specific (uses dopen / dread). - - Credit for the rename approach: - https://communities.sas.com/t5/SAS-Programming/SAS-Function-to-convert-string-to-Legal-SAS-Name/m-p/27375/highlight/true#M5003 - - Usage: - - %mp_dirlist(path=/some/location, outds=myTable, maxdepth=MAX) - - %mp_dirlist(outds=cwdfileprops, getattrs=YES) - - %mp_dirlist(fref=MYFREF) - - @warning In a Unix environment, the existence of a named pipe will cause this - macro to hang. Therefore this tool should be used with caution in a SAS 9 web - application, as it can use up all available multibridge sessions if requests - are resubmitted. - If anyone finds a way to positively identify a named pipe using SAS (without - X CMD) do please raise an issue! - - - @param [in] path= (%sysfunc(pathname(work))) Path for which to return contents - @param [in] fref= (0) Provide a DISK engine fileref as an alternative to PATH - @param [in] maxdepth= (0) Set to a positive integer to indicate the level of - subdirectory scan recursion - eg 3, to go `./3/levels/deep`. For unlimited - recursion, set to MAX. - @param [in] showparent= (NO) By default, the initial parent directory is not - part of the results. Set to YES to include it. For this record only, - directory=filepath. - @param [out] outds= (work.mp_dirlist) The output dataset to create - @param [out] getattrs= (NO) If getattrs=YES then the doptname / foptname - functions are used to scan all properties - any characters that are not - valid in a SAS name (v7) are simply stripped, and the table is transposed - so theat each property is a column and there is one file per row. An - attempt is made to get all properties whether a file or folder, but some - files/folders cannot be accessed, and so not all properties can / will be - populated. - - - @returns outds contains the following variables: - - directory (containing folder) - - file_or_folder (file / folder) - - filepath (path/to/file.name) - - filename (just the file name) - - ext (.extension) - - msg (system message if any issues) - - level (depth of folder) - - OS SPECIFIC variables, if getattrs= is used. - -

SAS Macros

- @li mf_existds.sas - @li mf_getvarlist.sas - @li mf_wordsinstr1butnotstr2.sas - @li mp_dropmembers.sas - -

Related Macros

- @li mp_dirlist.test.sas - - @version 9.2 -**//*** HELP END ***/ - -%macro mp_dirlist(path=%sysfunc(pathname(work)) - , fref=0 - , outds=work.mp_dirlist - , getattrs=NO - , showparent=NO - , maxdepth=0 - , level=0 /* The level of recursion to perform. For internal use only. */ -)/*/STORE SOURCE*/; -%let getattrs=%upcase(&getattrs)XX; - -/* temp table */ -%local out_ds; -data;run; -%let out_ds=%str(&syslast); - -/* drop main (top) table if it exists */ -%if &level=0 %then %do; - %mp_dropmembers(%scan(&outds,-1,.), libref=WORK) -%end; - -data &out_ds(compress=no - keep=file_or_folder filepath filename ext msg directory level - ); - length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80 - ext $20 msg $200 foption $16; - if _n_=1 then call missing(of _all_); - retain level &level; - %if &fref=0 %then %do; - rc = filename(fref, "&path"); - %end; - %else %do; - fref="&fref"; - rc=0; - %end; - if rc = 0 then do; - did = dopen(fref); - if did=0 then do; - putlog "NOTE: This directory is empty, or does not exist - &path"; - msg=sysmsg(); - put (_all_)(=); - stop; - end; - /* attribute is OS-dependent - could be "Directory" or "Directory Name" */ - numopts=doptnum(did); - do i=1 to numopts; - foption=doptname(did,i); - if foption=:'Directory' then i=numopts; - end; - directory=dinfo(did,foption); - rc = filename(fref); - end; - else do; - msg=sysmsg(); - put _all_; - stop; - end; - dnum = dnum(did); - do i = 1 to dnum; - filename = dread(did, i); - filepath=cats(directory,'/',filename); - rc = filename(fref2,filepath); - midd=dopen(fref2); - dmsg=sysmsg(); - if did > 0 then file_or_folder='folder'; - rc=dclose(midd); - midf=fopen(fref2); - fmsg=sysmsg(); - if midf > 0 then file_or_folder='file'; - rc=fclose(midf); - - if index(fmsg,'File is in use') or index(dmsg,'is not a directory') - then file_or_folder='file'; - else if index(fmsg,'Insufficient authorization') then file_or_folder='file'; - else if file_or_folder='' then file_or_folder='locked'; - - if file_or_folder='file' then do; - ext = prxchange('s/.*\.{1,1}(.*)/$1/', 1, filename); - if filename = ext then ext = ' '; - end; - else do; - ext=''; - file_or_folder='folder'; - end; - output; - end; - rc = dclose(did); - %if &showparent=YES and &level=0 %then %do; - filepath=directory; - file_or_folder='folder'; - ext=''; - filename=scan(directory,-1,'/\'); - msg=''; - level=&level; - output; - %end; - stop; -run; - -%if %substr(&getattrs,1,1)=Y %then %do; - data &out_ds; - set &out_ds; - length infoname infoval $60 fref $8; - if _n_=1 then call missing(fref); - rc=filename(fref,filepath); - drop rc infoname fid i close fref; - if file_or_folder='file' then do; - fid=fopen(fref); - if fid le 0 then do; - msg=sysmsg(); - putlog "Could not open file:" filepath fid= ; - sasname='_MCNOTVALID_'; - output; - end; - else do i=1 to foptnum(fid); - infoname=foptname(fid,i); - infoval=finfo(fid,infoname); - sasname=compress(infoname, '_', 'adik'); - if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname)); - if upcase(sasname) ne 'FILENAME' then output; - end; - close=fclose(fid); - end; - else do; - fid=dopen(fref); - if fid le 0 then do; - msg=sysmsg(); - putlog "Could not open folder:" filepath fid= ; - sasname='_MCNOTVALID_'; - output; - end; - else do i=1 to doptnum(fid); - infoname=doptname(fid,i); - infoval=dinfo(fid,infoname); - sasname=compress(infoname, '_', 'adik'); - if anydigit(sasname)=1 then sasname=substr(sasname,anyalpha(sasname)); - if upcase(sasname) ne 'FILENAME' then output; - end; - close=dclose(fid); - end; - run; - proc sort; - by filepath sasname; - proc transpose data=&out_ds out=&out_ds(drop=_:); - id sasname; - var infoval; - by filepath file_or_folder filename ext ; - run; -%end; - -data &out_ds; - set &out_ds(where=(filepath ne '')); -run; - -/** - * The above transpose can mean that some updates create additional columns. - * This necessitates the occasional use of datastep over proc append. - */ -%if %mf_existds(&outds) %then %do; - %local basevars appvars newvars; - %let basevars=%mf_getvarlist(&outds); - %let appvars=%mf_getvarlist(&out_ds); - %let newvars=%length(%mf_wordsinstr1butnotstr2(Str1=&appvars,Str2=&basevars)); - %if &newvars>0 %then %do; - data &outds; - set &outds &out_ds; - run; - %end; - %else %do; - proc append base=&outds data=&out_ds force nowarn; - run; - %end; -%end; -%else %do; - proc append base=&outds data=&out_ds; - run; -%end; - -/* recursive call */ -%if &maxdepth>&level or &maxdepth=MAX %then %do; - data _null_; - set &out_ds; - where file_or_folder='folder'; - %if &showparent=YES and &level=0 %then %do; - if filepath ne directory; - %end; - length code $10000; - code=cats('%nrstr(%mp_dirlist(path=',filepath,",outds=&outds" - ,",getattrs=&getattrs,level=%eval(&level+1),maxdepth=&maxdepth))"); - put code=; - call execute(code); - run; -%end; - -/* tidy up */ -proc sql; -drop table &out_ds; - -%mend mp_dirlist; diff --git a/005_macros/mp_ds2csv.sas b/005_macros/mp_ds2csv.sas deleted file mode 100644 index 4ee1529..0000000 --- a/005_macros/mp_ds2csv.sas +++ /dev/null @@ -1,212 +0,0 @@ -/*** HELP START ***//** - @file - @brief Export a dataset to a CSV file WITH leading blanks - @details Export a dataset to a file or fileref, retaining leading blanks. - - When using SASJS headerformat, the input statement is provided in the first - row of the CSV. - - Usage: - - %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv") - - filename example temp; - %mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS) - data; infile example; input;put _infile_; if _n_>5 then stop;run; - - data _null_; - infile example; - input; - call symputx('stmnt',_infile_); - stop; - run; - data work.want; - infile example dsd firstobs=2; - input &stmnt; - run; - - Why use mp_ds2csv over, say, proc export? - - 1. Ability to retain leading blanks (this is a major one) - 2. Control the header format - 3. Simple one-liner - - @param [in] ds The dataset to be exported - @param [in] dlm= (COMMA) The delimeter to apply. For SASJS, will always be - COMMA. Supported values: - @li COMMA - @li SEMICOLON - @param [in] headerformat= (LABEL) The format to use for the header section. - Valid values: - @li LABEL - Use the variable label (or name, if blank) - @li NAME - Use the variable name - @li SASJS - Used to create sasjs-formatted input CSVs, eg for use in - mp_testservice.sas. This format will supply an input statement in the - first row, making ingestion by datastep a breeze. Special misisng values - will be prefixed with a period (eg `.A`) to enable ingestion on both SAS 9 - and Viya. Dates / Datetimes etc are identified by the format type (lookup - with mcf_getfmttype.sas) and converted to human readable formats (not - numbers). - @param [out] outfile= The output filename - should be quoted. - @param [out] outref= (0) The output fileref (takes precedence if provided) - @param [in] outencoding= (0) The (quoted) output encoding to use, eg `"UTF-8"` - @param [in] termstr= (CRLF) The line seperator to use. For SASJS, will - always be CRLF. Valid values: - @li CRLF - @li LF - -

SAS Macros

- @li mcf_getfmttype.sas - @li mf_getuniquename.sas - @li mf_getvarformat.sas - @li mf_getvarlist.sas - @li mf_getvartype.sas - - @version 9.2 - @author Allan Bowe (credit mjsq) -**//*** HELP END ***/ - -%macro mp_ds2csv(ds - ,dlm=COMMA - ,outref=0 - ,outfile= - ,outencoding=0 - ,headerformat=LABEL - ,termstr=CRLF -)/*/STORE SOURCE*/; - -%local outloc delim i varlist var vcnt vat dsv vcom vmiss fmttype vfmt; - -%if not %sysfunc(exist(&ds)) %then %do; - %put %str(WARN)ING: &ds does not exist; - %return; -%end; - -%if %index(&ds,.)=0 %then %let ds=WORK.&ds; - -%if &outencoding=0 %then %let outencoding=; -%else %let outencoding=encoding=&outencoding; - -%if &outref=0 %then %let outloc=&outfile; -%else %let outloc=&outref; - -%if &headerformat=SASJS %then %do; - %let delim=","; - %let termstr=CRLF; - %mcf_getfmttype(wrap=YES) -%end; -%else %if &dlm=COMMA %then %let delim=","; -%else %let delim=";"; - -/* credit to mjsq - https://stackoverflow.com/a/55642267 */ - -/* first get headers */ -data _null_; - file &outloc &outencoding lrecl=32767 termstr=&termstr; - length header $ 2000 varnm vfmt $32 dlm $1 fmttype $8; - call missing(of _all_); - dsid=open("&ds.","i"); - num=attrn(dsid,"nvars"); - dlm=&delim; - do i=1 to num; - varnm=upcase(varname(dsid,i)); - if i=num then dlm=''; - %if &headerformat=NAME %then %do; - header=cats(varnm,dlm); - %end; - %else %if &headerformat=LABEL %then %do; - header = cats(coalescec(varlabel(dsid,i),varnm),dlm); - %end; - %else %if &headerformat=SASJS %then %do; - if vartype(dsid,i)='C' then header=cats(varnm,':$char',varlen(dsid,i),'.'); - else do; - vfmt=coalescec(varfmt(dsid,i),'0'); - fmttype=mcf_getfmttype(vfmt); - if fmttype='DATE' then header=cats(varnm,':date9.'); - else if fmttype='DATETIME' then header=cats(varnm,':E8601DT26.6'); - else if fmttype='TIME' then header=cats(varnm,':TIME12.'); - else header=cats(varnm,':best.'); - end; - %end; - %else %do; - %put &sysmacroname: Invalid headerformat value (&headerformat); - %return; - %end; - put header @; - end; - rc=close(dsid); -run; - -%let varlist=%mf_getvarlist(&ds); -%let vcnt=%sysfunc(countw(&varlist)); - -/** - * The $quote modifier (without a width) will take the length from the variable - * and increase by two. However this will lead to truncation where the value - * contains double quotes (which are doubled up). To get around this, scan the - * data to see the max number of double quotes, so that the appropriate width - * can be applied in the subsequent step. - */ -data _null_; - set &ds end=last; -%do i=1 %to &vcnt; - %let var=%scan(&varlist,&i); - %if %mf_getvartype(&ds,&var)=C %then %do; - %let dsv1=%mf_getuniquename(prefix=csvcol1_); - %let dsv2=%mf_getuniquename(prefix=csvcol2_); - retain &dsv1 0; - &dsv2=length(&var)+countc(&var,'"'); - if &dsv2>&dsv1 then &dsv1=&dsv2; - if last then call symputx( - "vlen&i" - /* should be no shorter than varlen, and no longer than 32767 */ - ,cats('$quote',min(&dsv1+2,32767),'.') - ,'l' - ); - %end; -%end; - -%let vat=@; -%let vcom=&delim; -%let vmiss=%mf_getuniquename(prefix=csvcol3_); -/* next, export data */ -data _null_; - set &ds.; - file &outloc mod dlm=&delim dsd &outencoding lrecl=32767 termstr=&termstr; - if _n_=1 then &vmiss=' '; - %do i=1 %to &vcnt; - %let var=%scan(&varlist,&i); - %if &i=&vcnt %then %do; - %let vat=; - %let vcom=; - %end; - %if %mf_getvartype(&ds,&var)=N %then %do; - %if &headerformat = SASJS %then %do; - %let vcom=&delim; - %let fmttype=%sysfunc(mcf_getfmttype(%mf_getvarformat(&ds,&var)0)); - %if &fmttype=DATE %then %let vfmt=DATE9.; - %else %if &fmttype=DATETIME %then %let vfmt=E8601DT26.6; - %else %if &fmttype=TIME %then %let vfmt=TIME12.; - %else %do; - %let vfmt=; - %let vcom=; - %end; - %end; - %else %let vcom=; - - /* must use period - in order to work in both 9.4 and Viya 3.5 */ - if missing(&var) and &var ne %sysfunc(getoption(MISSING)) then do; - &vmiss=cats('.',&var); - put &vmiss &vat; - end; - else put &var &vfmt &vcom &vat; - - %end; - %else %do; - %if &i ne &vcnt %then %let vcom=&delim; - put &var &&vlen&i &vcom &vat; - %end; - %end; -run; - -%mend mp_ds2csv; diff --git a/005_macros/mp_filtercheck.sas b/005_macros/mp_filtercheck.sas deleted file mode 100644 index 9a4bbff..0000000 --- a/005_macros/mp_filtercheck.sas +++ /dev/null @@ -1,253 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks an input filter table for validity - @details Performs checks on the input table to ensure it arrives in the - correct format. This is necessary to prevent code injection. Will update - SYSCC to 1008 if bad records are found, and call mp_abort.sas for a - graceful service exit (configurable). - - Used for dynamic filtering in [Data Controller for SAS®]( - https://datacontroller.io). - - Usage: - - %mp_filtercheck(work.filter,targetds=sashelp.class,outds=work.badrecords) - - The input table should have the following format: - - |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000| - |---|---|---|---|---|---| - |AND|AND|1|AGE|=|12| - |AND|AND|1|SEX|<=|'M'| - |AND|OR|2|Name|NOT IN|('Jane','Alfred')| - |AND|OR|2|Weight|>=|7| - - Rules applied: - - @li GROUP_LOGIC - only AND/OR - @li SUBGROUP_LOGIC - only AND/OR - @li SUBGROUP_ID - only integers - @li VARIABLE_NM - must be in the target table - @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS - @li RAW_VALUE - no unquoted values except integers, commas and spaces. - - @returns The &outds table containing any bad rows, plus a REASON_CD column. - - @param [in] inds The table to be checked, with the format above - @param [in] targetds= The target dataset against which to verify VARIABLE_NM. - This must be available (ie, the library must be assigned). - @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions - @param [out] outds= (work.badrecords) The output table, which is a copy of the - &inds. table plus a REASON_CD column, containing only bad records. - If bad records are found, the SYSCC value will be set to 1008 - (a general data problem). - Downstream processes should check this table (and return code) before - continuing. - -

SAS Macros

- @li mp_abort.sas - @li mf_getuniquefileref.sas - @li mf_getvarlist.sas - @li mf_getvartype.sas - @li mp_filtergenerate.sas - @li mp_filtervalidate.sas - -

Related Macros

- @li mp_filtergenerate.sas - @li mp_filtervalidate.sas - - @version 9.3 - @author Allan Bowe - - @todo Support date / hex / name literals and exponents in RAW_VALUE field -**//*** HELP END ***/ - -%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc - on macro entry) -) - -/* Validate input column */ -%local vtype; -%let vtype=%mf_getvartype(&inds,RAW_VALUE); -%mp_abort(iftrue=(&abort=YES and &vtype ne C), - mac=&sysmacroname, - msg=%str(%str(ERR)OR: RAW_VALUE must be character) -) -%if &vtype ne C %then %do; - %put &sysmacroname: RAW_VALUE must be character; - %let syscc=42; - %return; -%end; - - -/** - * Sanitise the values based on valid value lists, then strip out - * quotes, commas, periods and spaces. - * Only numeric values should remain - */ -%local reason_cd nobs; -%let nobs=0; -data &outds; - /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 - OPERATOR_NM $10 RAW_VALUE $4000;*/ - set &inds; - length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000; - drop tmp; - - /* quick check to ensure column exists */ - if upcase(VARIABLE_NM) not in - (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE))) - then do; - REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds"; - putlog REASON_CD= VARIABLE_NM=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - return; - end; - - /* need to open the dataset to get the column type */ - dsid=open("&targetds","i"); - if dsid>0 then do; - vnum=varnum(dsid,VARIABLE_NM); - if vnum<1 then do; - /* should not happen as was also tested for above */ - REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds"); - putlog REASON_CD= dsid=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - return; - end; - /* now we can get the type */ - else vtype=vartype(dsid,vnum); - end; - - /* closed list checks */ - if GROUP_LOGIC not in ('AND','OR') then do; - REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC); - putlog REASON_CD= GROUP_LOGIC=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - if SUBGROUP_LOGIC not in ('AND','OR') then do; - REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC); - putlog REASON_CD= SUBGROUP_LOGIC=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - if mod(SUBGROUP_ID,1) ne 0 then do; - REASON_CD='SUBGROUP_ID should be integer, not '!!cats(subgroup_id); - putlog REASON_CD= SUBGROUP_ID=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - if OPERATOR_NM not in - ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS') - then do; - REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM); - putlog REASON_CD= OPERATOR_NM=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - - /* special missing logic */ - if vtype='N' - and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE') - and cats(upcase(raw_value)) in ( - '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N' - '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._' - ) - then do; - /* valid numeric - exit data step loop */ - return; - end; - - /* special logic */ - if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do; - if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',','); - else do; - if substr(raw_value,1,1) ne '(' - or substr(cats(reverse(raw_value)),1,1) ne ')' - then do; - REASON_CD='Missing start/end bracket in RAW_VALUE'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0)); - end; - /* we now have a comma seperated list of values */ - if vtype='N' then do i=1 to countc(raw_value1, ',')+1; - tmp=scan(raw_value1,i,','); - if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - return; - end; - end; - else raw_value1=raw_value; - - /* remove nested literals eg '' */ - raw_value1=tranwrd(raw_value1,"''",''); - - /* now match string literals (always single quotes) */ - raw_value2=raw_value1; - regex = prxparse("s/(\').*?(\')//"); - call prxchange(regex,-1,raw_value2); - - /* remove commas and periods*/ - raw_value3=compress(raw_value2,',.'); - - /* output records that contain values other than digits and spaces */ - if notdigit(compress(raw_value3,' '))>0 then do; - putlog raw_value3= $hex32.; - REASON_CD=cats('Invalid RAW_VALUE:',raw_value); - putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; - -run; - - -data _null_; - set &outds end=last; - putlog (_all_)(=); -run; - -%mp_abort(iftrue=(&abort=YES and &nobs>0), - mac=&sysmacroname, - msg=%str(Data issue: %superq(reason_cd)) -) - -%if &nobs>0 %then %do; - %let syscc=1008; - %return; -%end; - -/** - * syntax checking passed but it does not mean the filter is valid - * for that we can run a proc sql validate query - */ -%local fref1; -%let fref1=%mf_getuniquefileref(); -%mp_filtergenerate(&inds,outref=&fref1) - -/* this macro will also set syscc to 1008 if any issues found */ -%mp_filtervalidate(&fref1,&targetds,outds=&outds,abort=&abort) - -%mend mp_filtercheck; diff --git a/005_macros/mp_loadformat.sas b/005_macros/mp_loadformat.sas deleted file mode 100644 index 8b40a3c..0000000 --- a/005_macros/mp_loadformat.sas +++ /dev/null @@ -1,363 +0,0 @@ -/*** HELP START ***//** - @file - @brief Loads a format catalog from a staging dataset - @details When loading staged data, it is common to receive only the records - that have actually changed. However, when loading a format catalog, if - records are missing they are presumed to be no longer required. - - This macro will augment a staging dataset with other records from the same - format, to prevent loss of data - UNLESS the input dataset contains a marker - column, specifying that a particular row needs to be deleted (`delete_col=`). - - This macro can also be used to identify which records would be (or were) - considered new, modified or deleted (`loadtarget=`) by creating the following - tables: - - @li work.outds_add - @li work.outds_del - @li work.outds_mod - - For example usage, see mp_loadformat.test.sas - - @param [in] libcat The format catalog to be loaded - @param [in] libds The staging table to load - @param [in] loadtarget= (NO) Set to YES to actually load the target catalog - @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to - mark a record for deletion. Values should be "Yes" or "No". - @param [out] auditlibds= (0) For change tracking, set to the libds of an audit - table as defined in mddl_dc_difftable.sas - @param [in] locklibds= (0) For multi-user (parallel) situations, set to the - libds of the DC lock table as defined in the mddl_dc_locktable.sas macro. - @param [out] outds_add= (0) Set a libds here to see the new records added - @param [out] outds_del= (0) Set a libds here to see the records deleted - @param [out] outds_mod= (0) Set a libds here to see the modified records - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_nobs.sas - @li mp_abort.sas - @li mp_aligndecimal.sas - @li mp_cntlout.sas - @li mp_lockanytable.sas - @li mp_storediffs.sas - -

Related Macros

- @li mddl_dc_difftable.sas - @li mddl_dc_locktable.sas - @li mp_loadformat.test.1.sas - @li mp_loadformat.test.2.sas - @li mp_lockanytable.sas - @li mp_stackdiffs.sas - - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_loadformat(libcat,libds - ,loadtarget=NO - ,auditlibds=0 - ,locklibds=0 - ,delete_col=_____DELETE__THIS__RECORD_____ - ,outds_add=0 - ,outds_del=0 - ,outds_mod=0 - ,mdebug=0 -); -/* set up local macro variables and temporary tables (with a prefix) */ -%local err msg prefix dslist i var fmtlist ibufsize; -%let dslist=base_fmts template inlibds ds1 stagedata storediffs del1 del2; -%if &outds_add=0 %then %let dslist=&dslist outds_add; -%if &outds_del=0 %then %let dslist=&dslist outds_del; -%if &outds_mod=0 %then %let dslist=&dslist outds_mod; -%let prefix=%substr(%mf_getuniquename(),1,21); -%do i=1 %to %sysfunc(countw(&dslist)); - %let var=%scan(&dslist,&i); - %local &var; - %let &var=%upcase(&prefix._&var); -%end; - -/* in DC, format catalogs maybe specified in the libds with a -FC extension */ -%let libcat=%scan(&libcat,1,-); - -/* perform input validations */ -%let err=0; -%let msg=0; -data _null_; - if _n_=1 then putlog "&sysmacroname entry vars:"; - set sashelp.vmacro; - where scope="&sysmacroname"; - value=upcase(value); - if &mdebug=0 then put name '=' value; - if name=:'LOAD' and value not in ('YES','NO') then do; - call symputx('msg',"invalid value for "!!name!!":"!!value); - call symputx('err',1); - stop; - end; - else if name='LIBCAT' then do; - if exist(value,'CATALOG') le 0 then do; - call symputx('msg',"Unable to open catalog: "!!value); - call symputx('err',1); - stop; - end; - end; - else if name='LIBDS' then do; - if exist(value) le 0 then do; - call symputx('msg',"Unable to open staging table: "!!value); - call symputx('err',1); - stop; - end; - end; - else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS')) - and missing(value) then do; - call symputx('msg',"missing value in var: "!!name); - call symputx('err',1); - stop; - end; -run; - -%mp_abort( - iftrue=(&err ne 0) - ,mac=&sysmacroname - ,msg=%str(&msg) -) - -/** - * First, extract only relevant formats from the catalog - */ -proc sql noprint; -select distinct - case - when type='N' then upcase(fmtname) - when type='C' then cats('$',upcase(fmtname)) - when type='I' then cats('@',upcase(fmtname)) - when type='J' then cats('@$',upcase(fmtname)) - else "&sysmacroname:UNHANDLED" - end - into: fmtlist separated by ' ' - from &libds; - -%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts) - -/* get a hash of the row */ -%local cvars nvars; -%let cvars=TYPE FMTNAME START END LABEL PREFIX FILL SEXCL EEXCL HLO DECSEP - DIG3SEP DATATYPE LANGUAGE; -%let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT; -data &base_fmts/note2err; - set &base_fmts; - fmthash=%mp_md5(cvars=&cvars, nvars=&nvars); -run; - -/** - * Ensure input table and base_formats have consistent lengths and types - */ -data &inlibds/nonote2err; - length &delete_col $3 FMTROW 8 start end label $32767; - if 0 then set &base_fmts; - set &libds; - by type fmtname notsorted; - if &delete_col='' then &delete_col='No'; - fmtname=upcase(fmtname); - type=upcase(type); - if missing(type) then do; - if substr(fmtname,1,1)='@' then do; - if substr(fmtname,2,1)='$' then type='J'; - else type='I'; - end; - else do; - if substr(fmtname,1,1)='$' then type='C'; - else type='N'; - end; - end; - if type in ('N','I') then do; - %mp_aligndecimal(start,width=16) - %mp_aligndecimal(end,width=16) - end; - - /* update row marker - retain new var as fmtrow may already be in libds */ - if first.fmtname then row=1; - else row+1; - drop row; - fmtrow=row; - - fmthash=%mp_md5(cvars=&cvars, nvars=&nvars); -run; - -/** - * Identify new records - */ -proc sql; -create table &outds_add(drop=&delete_col) as - select a.* - from &inlibds a - left join &base_fmts b - on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where b.fmtname is null - and upcase(a.&delete_col) ne "YES" - order by type, fmtname, fmtrow; - -/** - * Identify modified records - */ -create table &outds_mod (drop=&delete_col) as - select a.* - from &inlibds a - inner join &base_fmts b - on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where upcase(a.&delete_col) ne "YES" - and a.fmthash ne b.fmthash - order by type, fmtname, fmtrow; - -/** - * Identify deleted records - */ -create table &outds_del(drop=&delete_col) as - select a.* - from &inlibds a - inner join &base_fmts b - on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where upcase(a.&delete_col)="YES" - order by type, fmtname, fmtrow; - -/** - * Identify fully deleted formats (where every record is removed) - * These require to be explicitly deleted in proc format - * del1 - identify _partial_ deletes - * del2 - exclude these, and also formats that come with _additions_ - */ -create table &del1 as - select a.* - from &base_fmts a - left join &outds_del b - on a.type=b.type and a.fmtname=b.fmtname and a.fmtrow=b.fmtrow - where b.fmtrow is null; - -create table &del2 as - select * from &outds_del - where cats(type,fmtname) not in (select cats(type,fmtname) from &outds_add) - and cats(type,fmtname) not in (select cats(type,fmtname) from &del1); - - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(SYSCC=&syscc prior to load prep) -) - -%if &loadtarget=YES %then %do; - /* new records plus base records that are not deleted or modified */ - data &ds1; - merge &base_fmts(in=base) - &outds_mod(in=mod) - &outds_add(in=add) - &outds_del(in=del); - if not del and not mod; - by type fmtname fmtrow; - run; - /* add back the modified records */ - data &stagedata; - set &ds1 &outds_mod; - run; - proc sort; - by type fmtname fmtrow; - run; -%end; -/* mp abort needs to run outside of conditional blocks */ -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(SYSCC=&syscc prior to actual load) -) -%if &loadtarget=YES %then %do; - %if %mf_nobs(&stagedata)=0 and %mf_nobs(&del2)=0 %then %do; - %put There are no changes to load in &libcat!; - %return; - %end; - %if &locklibds ne 0 %then %do; - /* prevent parallel updates */ - %mp_lockanytable(LOCK - ,lib=%scan(&libcat,1,.) - ,ds=%scan(&libcat,2,.)-FC - ,ref=MP_LOADFORMAT commencing format load - ,ctl_ds=&locklibds - ) - %end; - /* do the actual load */ - proc format lib=&libcat cntlin=&stagedata; - run; - /* apply any full deletes */ - %if %mf_nobs(&del2)>0 %then %do; - %local delfmtlist; - proc sql noprint; - select distinct case when type='N' then cats(fmtname,'.FORMAT') - when type='C' then cats(fmtname,'.FORMATC') - when type='J' then cats(fmtname,'.INFMTC') - when type='I' then cats(fmtname,'.INFMT') - else cats(fmtname,'.BADENTRY!!!') end - into: delfmtlist - separated by ' ' - from &del2; - proc catalog catalog=&libcat; - delete &delfmtlist; - quit; - %end; - %if &locklibds ne 0 %then %do; - /* unlock the table */ - %mp_lockanytable(UNLOCK - ,lib=%scan(&libcat,1,.) - ,ds=%scan(&libcat,2,.)-FC - ,ref=MP_LOADFORMAT completed format load - ,ctl_ds=&locklibds - ) - %end; - /* track the changes */ - %if &auditlibds ne 0 %then %do; - %if &locklibds ne 0 %then %do; - %mp_lockanytable(LOCK - ,lib=%scan(&auditlibds,1,.) - ,ds=%scan(&auditlibds,2,.) - ,ref=MP_LOADFORMAT commencing audit table load - ,ctl_ds=&locklibds - ) - %end; - - %mp_storediffs(&libcat-FC - ,&base_fmts - ,TYPE FMTNAME FMTROW - ,delds=&outds_del - ,modds=&outds_mod - ,appds=&outds_add - ,outds=&storediffs - ,mdebug=&mdebug - ) - - proc append base=&auditlibds data=&storediffs; - run; - - %if &locklibds ne 0 %then %do; - %mp_lockanytable(UNLOCK - ,lib=%scan(&auditlibds,1,.) - ,ds=%scan(&auditlibds,2,.) - ,ref=MP_LOADFORMAT commencing audit table load - ,ctl_ds=&locklibds - ) - %end; - %end; -%end; -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(SYSCC=&syscc after load) -) - -%if &mdebug=0 %then %do; - proc datasets lib=work; - delete &prefix:; - run; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%mend mp_loadformat; diff --git a/005_macros/mp_stackdiffs.sas b/005_macros/mp_stackdiffs.sas deleted file mode 100644 index 24bb88b..0000000 --- a/005_macros/mp_stackdiffs.sas +++ /dev/null @@ -1,592 +0,0 @@ -/*** HELP START ***//** - @file - @brief Prepares an audit table for stacking (re-applying) the changes. - @details When the underlying data from a Base Table is refreshed, it can be - helpful to have any previously-applied changes, re-applied. - - Such situation might arise if you are applying those changes using a tool - like [Data Controller for SAS®](https://datacontroller.io) - which records - all such changes in an audit table. - It may also apply if you are preparing a series of specific cell-level - transactions, that you would like to apply to multiple sets of (similarly - structured) Base Tables. - - In both cases, it is necessary that the transactions are stored using - the mp_storediffs.sas macro, or at least that the underlying table is - structured as per the definition in mp_coretable.sas (DIFFTABLE entry) - - This macro is used to convert the stored changes (tall format) into - staged changes (wide format), with base table values incorporated (in the - case of modified rows), ready for the subsequent load process. - - Essentially then, what this macro does, is turn a table like this: - -|KEY_HASH:$32.|MOVE_TYPE:$1.|TGTVAR_NM:$32.|IS_PK:best.|IS_DIFF:best.|TGTVAR_TYPE:$1.|OLDVAL_NUM:best32.|NEWVAL_NUM:best32.|OLDVAL_CHAR:$32765.|NEWVAL_CHAR:$32765.| -|---|---|---|---|---|---|---|---|---|---| -|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|` `|`Newbie `| -|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`AGE `|`0 `|`-1 `|`N `|`. `|`13 `|` `|` `| -|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`HEIGHT `|`0 `|`-1 `|`N `|`. `|`65.3 `|` `|` `| -|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|` `|`F `| -|`27AA6F7581052E7FF48E1BCA901313FB `|`A `|`WEIGHT `|`0 `|`-1 `|`N `|`. `|`98 `|` `|` `| -|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`NAME `|`1 `|`-1 `|`C `|`. `|`. `|`Alfred `|` `| -|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`AGE `|`0 `|`-1 `|`N `|`14 `|`. `|` `|` `| -|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`HEIGHT `|`0 `|`-1 `|`N `|`69 `|`. `|` `|` `| -|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`SEX `|`0 `|`-1 `|`C `|`. `|`. `|`M `|` `| -|`86703FDE9E87DD5C0F8E1072545D0128 `|`D `|`WEIGHT `|`0 `|`-1 `|`N `|`112.5 `|`. `|` `|` `| -|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`NAME `|`1 `|`0 `|`C `|`. `|`. `|`Alice `|`Alice `| -|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`AGE `|`0 `|`1 `|`N `|`13 `|`99 `|` `|` `| -|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`HEIGHT `|`0 `|`0 `|`N `|`56.5 `|`56.5 `|` `|` `| -|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`SEX `|`0 `|`0 `|`C `|`. `|`. `|`F `|`F `| -|`64489C85DC2FE0787B85CD87214B3810 `|`M `|`WEIGHT `|`0 `|`0 `|`N `|`84 `|`84 `|` `|` `| - - Into three tables like this: - - `work.outmod`: - |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| - |---|---|---|---|---| - |`Alice `|`F `|`99 `|`56.5 `|`84 `| - - `work.outadd`: - |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| - |---|---|---|---|---| - |`Newbie `|`F `|`13 `|`65.3 `|`98 `| - - `work.outdel`: - |NAME:$8.|SEX:$1.|AGE:best.|HEIGHT:best.|WEIGHT:best.| - |---|---|---|---|---| - |`Alfred `|`M `|`14 `|`69 `|`112.5 `| - - As you might expect, there are a bunch of extra features and checks. - - The macro supports both SCD2 (TXTEMPORAL) and UPDATE loadtypes. If the - base table contains a PROCESSED_DTTM column (or similar), this can be - ignored by declaring it in the `processed_dttm_var` parameter. - - The macro is also flexible where columns have been added or removed from - the base table UNLESS there is a change to the primary key. - - Changes to the primary key fields are NOT supported, and are likely to cause - unexpected results. - - The following pre-flight checks are made: - - @li All primary key columns exist on the base table - @li There is no change in variable TYPE for any of the columns - @li There is no reduction in variable LENGTH below the max-length of the - supplied values - - Rules for stacking changes are as follows: - - - - - - - - - - - - - - - - - - - - -
Transaction TypeKey BehaviourColumn Behaviour
Deletes - The row is added to `&outDEL.` UNLESS it no longer exists - in the base table, in which case it is added to `&errDS.` instead. - - Deletes are unaffected by the addition or removal of non Primary-Key - columns. -
Inserts - Previously newly added rows are added to the `outADD` table UNLESS they - are present in the Base table.
In this case they are added to the - `&errDS.` table instead. - 		- Inserts are unaffected by the addition of columns in the Base Table - (they are padded with blanks). Deleted columns are only a problem if - they appear on the previous insert - in which case the record is added - to `&errDS.`. -
Updates - Previously modified rows are merged with base table values such that - only the individual cells that were _previously_ changed are re-applied. - Where the row contains cells that were not marked as having changed in - the prior transaction, the 'blanks' are filled with base table values in - the `outMOD` table.
- If the row no longer exists on the base table, then the row is added to - the `errDS` table instead. - 		- Updates are unaffected by the addition of columns in the Base Table - - the new cells are simply populated with Base Table values. Deleted - columns are only a problem if they relate to a modified cell - (`is_diff=1`) - in which case the record is added to `&errDS.`. -
- - To illustrate the above with a diagram: - - @dot - digraph { - rankdir="TB" - start[label="Transaction Type?" shape=Mdiamond] - del[label="Does Base Row exist?" shape=rectangle] - add [label="Does Base Row exist?" shape=rectangle] - mod [label="Does Base Row exist?" shape=rectangle] - chkmod [label="Do all modified\n(is_diff=1) cells exist?" shape=rectangle] - chkadd [label="Do all inserted cells exist?" shape=rectangle] - outmod [label="outMOD\nTable" shape=Msquare style=filled] - outadd [label="outADD\nTable" shape=Msquare style=filled] - outdel [label="outDEL\nTable" shape=Msquare style=filled] - outerr [label="ErrDS Table" shape=Msquare fillcolor=Orange style=filled] - start -> del [label="Delete"] - start -> add [label="Insert"] - start -> mod [label="Update"] - - del -> outdel [label="Yes"] - del -> outerr [label="No" color="Red" fontcolor="Red"] - add -> chkadd [label="No"] - add -> outerr [label="Yes" color="Red" fontcolor="Red"] - mod -> outerr [label="No" color="Red" fontcolor="Red"] - mod -> chkmod [label="Yes"] - chkmod -> outerr [label="No" color="Red" fontcolor="Red"] - chkmod -> outmod [label="Yes"] - chkadd -> outerr [label="No" color="Red" fontcolor="Red"] - chkadd -> outadd [label="Yes"] - - } - @enddot - - For examples of usage, check out the mp_stackdiffs.test.sas program. - - - @param [in] baselibds Base Table against which the changes will be applied, - in libref.dataset format. - @param [in] auditlibds Dataset with previously applied transactions, to be - re-applied. Use libref.dataset format. - DDL as follows: %mp_coretable(DIFFTABLE) - @param [in] key Space seperated list of key variables - @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs - @param [in] processed_dttm_var= (0) If a variable is being used to mark - the processed datetime, put the name of the variable here. It will NOT - be included in the staged dataset (the load process is expected to - provide this) - @param [out] errds= (work.errds) Output table containing problematic records. - The columns of this table are: - @li PK_VARS - Space separated list of primary key variable names - @li PK_VALS - Slash separted list of PK variable values - @li ERR_MSG - Explanation of why this record is problematic - @param [out] outmod= (work.outmod) Output table containing modified records - @param [out] outadd= (work.outadd) Output table containing additional records - @param [out] outdel= (work.outdel) Output table containing deleted records - -

SAS Macros

- @li mf_existvarlist.sas - @li mf_getquotedstr.sas - @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mf_islibds.sas - @li mf_nobs.sas - @li mf_wordsinstr1butnotstr2.sas - @li mp_abort.sas - @li mp_ds2squeeze.sas - -

Related Macros

- @li mp_coretable.sas - @li mp_stackdiffs.test.sas - @li mp_storediffs.sas - - @todo The current approach assumes that a variable called KEY_HASH is not on - the base table. This part will need to be refactored (eg using - mf_getuniquename.sas) when such a use case arises. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ - -%macro mp_stackdiffs(baselibds - ,auditlibds - ,key - ,mdebug=0 - ,processed_dttm_var=0 - ,errds=work.errds - ,outmod=work.outmod - ,outadd=work.outadd - ,outdel=work.outdel -)/*/STORE SOURCE*/; -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -/* input parameter validations */ -%mp_abort(iftrue= (%mf_islibds(&baselibds) ne 1) - ,mac=&sysmacroname - ,msg=%str(Invalid baselibds: &baselibds) -) -%mp_abort(iftrue= (%mf_islibds(&auditlibds) ne 1) - ,mac=&sysmacroname - ,msg=%str(Invalid auditlibds: &auditlibds) -) -%mp_abort(iftrue= (%length(&key)=0) - ,mac=&sysmacroname - ,msg=%str(Missing key variables!) -) -%mp_abort(iftrue= ( - %mf_existVarList(&auditlibds,LIBREF DSN MOVE_TYPE KEY_HASH TGTVAR_NM IS_PK - IS_DIFF TGTVAR_TYPE OLDVAL_NUM NEWVAL_NUM OLDVAL_CHAR NEWVAL_CHAR)=0 - ) - ,mac=&sysmacroname - ,msg=%str(Input &auditlibds is missing required columns!) -) - - -/* set up macro vars */ -%local prefix dslist x var keyjoin commakey keepvars missvars fref; -%let prefix=%substr(%mf_getuniquename(),1,25); -%let dslist=ds1d ds2d ds3d ds1a ds2a ds3a ds1m ds2m ds3m pks dups base - delrec delerr addrec adderr modrec moderr; -%do x=1 %to %sysfunc(countw(&dslist)); - %let var=%scan(&dslist,&x); - %local &var; - %let &var=%upcase(&prefix._&var); -%end; - -%let key=%upcase(&key); -%let commakey=%mf_getquotedstr(&key,quote=N); - -%let keyjoin=1=1; -%do x=1 %to %sysfunc(countw(&key)); - %let var=%scan(&key,&x); - %let keyjoin=&keyjoin and a.&var=b.&var; -%end; - -data &errds; - length pk_vars $256 pk_vals $4098 err_msg $512; - call missing (of _all_); - stop; -run; - -/** - * Prepare raw DELETE table - * Records are in the OLDVAL_xxx columns - */ -%let keepvars=MOVE_TYPE KEY_HASH TGTVAR_NM TGTVAR_TYPE IS_PK - OLDVAL_NUM OLDVAL_CHAR - NEWVAL_NUM NEWVAL_CHAR; -proc sort data=&auditlibds(where=(move_type='D') keep=&keepvars) - out=&ds1d(drop=move_type); -by KEY_HASH TGTVAR_NM; -run; -proc transpose data=&ds1d(where=(tgtvar_type='N')) - out=&ds2d(drop=_name_); - by KEY_HASH; - id TGTVAR_NM; - var OLDVAL_NUM; -run; -proc transpose data=&ds1d(where=(tgtvar_type='C')) - out=&ds3d(drop=_name_); - by KEY_HASH; - id TGTVAR_NM; - var OLDVAL_CHAR; -run; -%mp_ds2squeeze(&ds2d,outds=&ds2d) -%mp_ds2squeeze(&ds3d,outds=&ds3d) -data &outdel; - if 0 then set &baselibds; - set &ds2d; - set &ds3d; - drop key_hash; - if not missing(%scan(&key,1)); -run; -proc sort; - by &key; -run; - -/** - * Prepare raw APPEND table - * Records are in the NEWVAL_xxx columns - */ -proc sort data=&auditlibds(where=(move_type='A') keep=&keepvars) - out=&ds1a(drop=move_type); - by KEY_HASH TGTVAR_NM; -run; -proc transpose data=&ds1a(where=(tgtvar_type='N')) - out=&ds2a(drop=_name_); - by KEY_HASH; - id TGTVAR_NM; - var NEWVAL_NUM; -run; -proc transpose data=&ds1a(where=(tgtvar_type='C')) - out=&ds3a(drop=_name_); - by KEY_HASH; - id TGTVAR_NM; - var NEWVAL_CHAR; -run; -%mp_ds2squeeze(&ds2a,outds=&ds2a) -%mp_ds2squeeze(&ds3a,outds=&ds3a) -data &outadd; - if 0 then set &baselibds; - set &ds2a; - set &ds3a; - drop key_hash; - if not missing(%scan(&key,1)); -run; -proc sort; - by &key; -run; - -/** - * Prepare raw MODIFY table - * Keep only primary key - will add modified values later - */ -proc sort data=&auditlibds( - where=(move_type='M' and is_pk=1) keep=&keepvars - ) out=&ds1m(drop=move_type); - by KEY_HASH TGTVAR_NM; -run; -proc transpose data=&ds1m(where=(tgtvar_type='N')) - out=&ds2m(drop=_name_); - by KEY_HASH ; - id TGTVAR_NM; - var NEWVAL_NUM; -run; -proc transpose data=&ds1m(where=(tgtvar_type='C')) - out=&ds3m(drop=_name_); - by KEY_HASH; - id TGTVAR_NM; - var NEWVAL_CHAR; -run; -%mp_ds2squeeze(&ds2m,outds=&ds2m) -%mp_ds2squeeze(&ds3m,outds=&ds3m) -data &outmod; - if 0 then set &baselibds; - set &ds2m; - set &ds3m; - if not missing(%scan(&key,1)); -run; -proc sort; - by &key; -run; - -/** - * Extract matching records from the base table - * Do this in one join for efficiency. - * At a later date, this should be optimised for large database tables by using - * passthrough and a temporary table. - */ -data &pks; - if 0 then set &baselibds; - set &outadd &outmod &outdel; - keep &key; -run; - -proc sort noduprec dupout=&dups; -by &key; -run; -data _null_; - set &dups; - putlog (_all_)(=); -run; -%mp_abort(iftrue= (%mf_nobs(&dups) ne 0) - ,mac=&sysmacroname - ,msg=%str(duplicates (%mf_nobs(&dups)) found on &auditlibds!) -) - -proc sql; -create table &base as - select a.* - from &baselibds a, &pks b - where &keyjoin; - -/** - * delete check - * This is straightforward as it relates to records only - */ -proc sql; -create table &delrec as - select a.* - from &outdel a - left join &base b - on &keyjoin - where b.%scan(&key,1) is null - order by &commakey; - -data &delerr; - if 0 then set &errds; - set &delrec; - PK_VARS="&key"; - PK_VALS=catx('/',&commakey); - ERR_MSG="Rows cannot be deleted as they do not exist on the Base dataset"; - keep PK_VARS PK_VALS ERR_MSG; -run; -proc append base=&errds data=&delerr; -run; - -data &outdel; - merge &outdel (in=a) &delrec (in=b); - by &key; - if not b; -run; - -/** - * add check - * Problems - where record already exists, or base table has columns missing - */ -%let missvars=%mf_wordsinstr1butnotstr2( - Str1=%upcase(%mf_getvarlist(&outadd)), - Str2=%upcase(%mf_getvarlist(&baselibds)) -); -%if %length(&missvars)>0 %then %do; - /* add them to the err table */ - data &adderr; - if 0 then set &errds; - set &outadd; - PK_VARS="&key"; - PK_VALS=catx('/',&commakey); - ERR_MSG="Rows cannot be added due to missing base vars: &missvars"; - keep PK_VARS PK_VALS ERR_MSG; - run; - proc append base=&errds data=&adderr; - run; - proc sql; - delete * from &outadd; -%end; -%else %do; - proc sql; - /* find records that already exist on base table */ - create table &addrec as - select a.* - from &outadd a - inner join &base b - on &keyjoin - order by &commakey; - - /* add them to the err table */ - data &adderr; - if 0 then set &errds; - set &addrec; - PK_VARS="&key"; - PK_VALS=catx('/',&commakey); - ERR_MSG="Rows cannot be added as they already exist on the Base dataset"; - keep PK_VARS PK_VALS ERR_MSG; - run; - proc append base=&errds data=&adderr; - run; - - /* remove invalid rows from the outadd table */ - data &outadd; - merge &outadd (in=a) &addrec (in=b); - by &key; - if not b; - run; -%end; - -/** - * mod check - * Problems - where record does not exist or baseds has modified cols missing - */ -proc sql noprint; -select distinct tgtvar_nm into: missvars separated by ' ' - from &auditlibds - where move_type='M' and is_diff=1; -%let missvars=%mf_wordsinstr1butnotstr2( - Str1=&missvars, - Str2=%upcase(%mf_getvarlist(&baselibds)) -); -%if %length(&missvars)>0 %then %do; - /* add them to the err table */ - data &moderr; - if 0 then set &errds; - set &outmod; - PK_VARS="&key"; - PK_VALS=catx('/',&commakey); - ERR_MSG="Rows cannot be modified due to missing base vars: &missvars"; - keep PK_VARS PK_VALS ERR_MSG; - run; - proc append base=&errds data=&moderr; - run; - proc sql; - delete * from &outmod; -%end; -%else %do; - /* now check for records that do not exist (therefore cannot be modified) */ - proc sql; - create table &modrec as - select a.* - from &outmod a - left join &base b - on &keyjoin - where b.%scan(&key,1) is null - order by &commakey; - data &moderr; - if 0 then set &errds; - set &modrec; - PK_VARS="&key"; - PK_VALS=catx('/',&commakey); - ERR_MSG="Rows cannot be modified as they do not exist on the Base dataset"; - keep PK_VARS PK_VALS ERR_MSG; - run; - proc append base=&errds data=&moderr; - run; - /* delete the above records from the outmod table */ - data &outmod; - merge &outmod (in=a) &modrec (in=b); - by &key; - if not b; - run; - /* now - we can prepare the final MOD table (which is currently PK only) */ - proc sql undo_policy=none; - create table &outmod as - select a.key_hash - ,b.* - from &outmod a - inner join &base b - on &keyjoin - order by &commakey; - /* now - to update outmod with modified (is_diff=1) values */ - %let fref=%mf_getuniquefileref(); - data _null_; - file &fref; - set &auditlibds(where=(move_type='M')) end=lastobs; - by key_hash; - retain comma 'N'; - if _n_=1 then put 'proc sql;'; - if first.key_hash then do; - comma='N'; - put "update &outmod set " @@; - end; - if is_diff=1 then do; - if comma='N' then do; - put ' '@@; - comma='Y'; - end; - else put ' ,'@@; - if tgtvar_type='C' then do; - length qstr $32767; - qstr=quote(trim(NEWVAL_CHAR)); - put tgtvar_nm '=' qstr; - end; - else put tgtvar_nm '=' newval_num; - if comma=' ' then comma=' ,'; - end; - if last.key_hash then put ' where key_hash=trim("' key_hash '");'; - if lastobs then put "alter table &outmod drop key_hash;"; - run; - %inc &fref/source2; -%end; - -%if &mdebug=0 %then %do; - proc datasets lib=work; - delete &prefix:; - run; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%mend mp_stackdiffs; -/** @endcond */ diff --git a/005_macros/mp_storediffs.sas b/005_macros/mp_storediffs.sas deleted file mode 100644 index ffad990..0000000 --- a/005_macros/mp_storediffs.sas +++ /dev/null @@ -1,229 +0,0 @@ -/*** HELP START ***//** - @file - @brief Converts deletes/changes/appends into a single audit table. - @details When tracking changes to data over time, it can be helpful to have - a single base table to track ALL modifications - enabling audit trail, - data recovery, and change re-application. This macro is one of many - data management utilities used in [Data Controller for SAS]( - https:datacontroller.io) - a comprehensive data ingestion solution, which - works on any SAS platform (Viya, SAS 9, Foundation). - - NOTE - this macro does not validate the inputs. It is assumed that the - datasets containing the new / changed / deleted rows are CORRECT, contain - no additional (or missing columns), and that the originals dataset contains - all relevant base records (and no additionals). - - Usage: - - data work.orig work.deleted work.changed work.appended; - set sashelp.class; - if _n_=1 then do; - output work.orig work.deleted; - end; - else if _n_=2 then do; - output work.orig; - age=99; - output work.changed; - end; - else do; - name='Newbie'; - output work.appended; - stop; - end; - run; - - %mp_storediffs(sashelp.class,work.orig,NAME - ,delds=work.deleted - ,modds=work.changed - ,appds=work.appended - ,outds=work.final - ,mdebug=1 - ) - - @param [in] libds Target table against which the changes were applied - @param [in] origds Dataset with original (unchanged) records. Can be empty if - only appending. - @param [in] key Space seperated list of key variables - @param [in] delds= (0) Dataset with deleted records - @param [in] appds= (0) Dataset with appended records - @param [in] modds= (0) Dataset with modified records - @param [out] outds= (work.mp_storediffs) Output table containing stored data. - DDL as follows: %mp_coretable(DIFFTABLE) - - @param [in] processed_dttm= (0) Provide a datetime constant in relation to - the actual load time. If not provided, current timestamp is used. - @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs - @param [out] loadref= (0) Provide a unique key to reference the load, - otherwise a UUID will be generated. - -

SAS Macros

- @li mf_getquotedstr.sas - @li mf_getuniquename.sas - @li mf_getvarlist.sas - -

Related Macros

- @li mp_stackdiffs.sas - @li mp_storediffs.test.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ - -%macro mp_storediffs(libds - ,origds - ,key - ,delds=0 - ,appds=0 - ,modds=0 - ,outds=work.mp_storediffs - ,loadref=0 - ,processed_dttm=0 - ,mdebug=0 -)/*/STORE SOURCE*/; -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -/* set up unique and temporary vars */ -%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist; -%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1)); -%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2)); -%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3)); -%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_ds4)); -%let hashkey=%upcase(%mf_getuniquename(prefix=mpsd_hashkey)); -%let inds_auto=%upcase(%mf_getuniquename(prefix=mpsd_inds_auto)); -%let inds_keep=%upcase(%mf_getuniquename(prefix=mpsd_inds_keep)); - -%let dslist=&origds; -%if &delds ne 0 %then %do; - %let delds=%upcase(&delds); - %if %scan(&delds,-1,.)=&delds %then %let delds=WORK.&delds; - %let dslist=&dslist &delds; -%end; -%if &appds ne 0 %then %do; - %let appds=%upcase(&appds); - %if %scan(&appds,-1,.)=&appds %then %let appds=WORK.&appds; - %let dslist=&dslist &appds; -%end; -%if &modds ne 0 %then %do; - %let modds=%upcase(&modds); - %if %scan(&modds,-1,.)=&modds %then %let modds=WORK.&modds; - %let dslist=&dslist &modds; -%end; - -%let origds=%upcase(&origds); -%if %scan(&origds,-1,.)=&origds %then %let origds=WORK.&origds; - -%let key=%upcase(&key); - -/* hash the key and append all the tables (marking the source) */ -data &ds1; - set &dslist indsname=&inds_auto; - &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.); - &inds_keep=upcase(&inds_auto); -proc sort; - by &inds_keep &hashkey; -run; - -/* transpose numeric & char vars */ -proc transpose data=&ds1 - out=&ds2(rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_num)); - by &inds_keep &hashkey; - var _numeric_; -run; -proc transpose data=&ds1 - out=&ds3( - rename=(&hashkey=key_hash _name_=tgtvar_nm col1=newval_char) - where=(tgtvar_nm not in ("&hashkey","&inds_keep")) - ); - by &inds_keep &hashkey; - var _character_; -run; - -%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do; - /* this is a format catalog - cannot query cols directly */ - %let vlist="TYPE","FMTNAME","FMTROW","START","END","LABEL","MIN","MAX" - ,"DEFAULT","LENGTH","FUZZ","PREFIX","MULT","FILL","NOEDIT","SEXCL" - ,"EEXCL","HLO","DECSEP","DIG3SEP","DATATYPE","LANGUAGE"; -%end; -%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE); - -data &ds4; - length &inds_keep $41 tgtvar_nm $32 _label_ $256; - if _n_=1 then call missing(_label_); - drop _label_; - set &ds2 &ds3 indsname=&inds_auto; - - tgtvar_nm=upcase(tgtvar_nm); - if tgtvar_nm in (%upcase(&vlist)); - - if upcase(&inds_auto)="&ds2" then tgtvar_type='N'; - else if upcase(&inds_auto)="&ds3" then tgtvar_type='C'; - else do; - putlog 'ERR' +(-1) "OR: unidentified vartype input!" &inds_auto; - call symputx('syscc',98); - end; - - if &inds_keep="&appds" then move_type='A'; - else if &inds_keep="&delds" then move_type='D'; - else if &inds_keep="&modds" then move_type='M'; - else if &inds_keep="&origds" then move_type='O'; - else do; - putlog 'ERR' +(-1) "OR: unidentified movetype input!" &inds_keep; - call symputx('syscc',99); - end; - tgtvar_nm=upcase(tgtvar_nm); - if tgtvar_nm in (%mf_getquotedstr(&key)) then is_pk=1; - else is_pk=0; - drop &inds_keep; -run; - -%if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen()); -%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime()); -%let libds=%upcase(&libds); - -/* join orig vals for modified & deleted */ -proc sql; -create table &outds as - select "&loadref" as load_ref length=36 - ,&processed_dttm as processed_dttm format=E8601DT26.6 - ,"%scan(&libds,1,.)" as libref length=8 - ,"%scan(&libds,2,.)" as dsn length=32 - ,b.key_hash length=32 - ,b.move_type length=1 - ,b.tgtvar_nm length=32 - ,b.is_pk - ,case when b.move_type ne 'M' then -1 - when a.newval_num=b.newval_num and a.newval_char=b.newval_char then 0 - else 1 - end as is_diff - ,b.tgtvar_type length=1 - ,case when b.move_type='D' then b.newval_num - else a.newval_num - end as oldval_num format=best32. - ,case when b.move_type='D' then . - else b.newval_num - end as newval_num format=best32. - ,case when b.move_type='D' then b.newval_char - else a.newval_char - end as oldval_char length=32765 - ,case when b.move_type='D' then '' - else b.newval_char - end as newval_char length=32765 - from &ds4(where=(move_type='O')) as a - right join &ds4(where=(move_type ne 'O')) as b - on a.tgtvar_nm=b.tgtvar_nm - and a.key_hash=b.key_hash - order by move_type, key_hash,is_pk desc, tgtvar_nm; - -%if &mdebug=0 %then %do; - proc sql; - drop table &ds1, &ds2, &ds3, &ds4; -%end; - -%mend mp_storediffs; -/** @endcond */ diff --git a/005_macros/mp_validatecol.sas b/005_macros/mp_validatecol.sas deleted file mode 100644 index 440501d..0000000 --- a/005_macros/mp_validatecol.sas +++ /dev/null @@ -1,97 +0,0 @@ -/*** HELP START ***//** - @file - @brief Used to validate variables in a dataset - @details Useful when sanitising inputs, to ensure that they arrive with a - certain pattern. - Usage: - - data test; - infile datalines4 dsd; - input; - libds=_infile_; - %mp_validatecol(libds,LIBDS,is_libds) - datalines4; - some.libname - !lib.blah - %abort - definite.ok - not.ok! - nineletrs._ - ;;;; - run; - - For more examples, see mp_validatecol.test.sas - - Tip - when contributing, use https://regex101.com to test the regex validity! - - @param [in] incol The column to be validated - @param [in] rule The rule to apply. Current rules: - @li ISINT - checks if the variable is an integer - @li ISNUM - checks if the variable is numeric - @li LIBDS - matches LIBREF.DATASET format - @li FORMAT - checks if the provided format is syntactically valid - @param [out] outcol The variable to create, with the results of the match - -

SAS Macros

- @li mf_getuniquename.sas - -

Related Macros

- @li mp_validatecol.test.sas - - @version 9.3 -**//*** HELP END ***/ - -%macro mp_validatecol(incol,rule,outcol); - -/* tempcol is given a unique name with every invocation */ -%local tempcol; -%let tempcol=%mf_getuniquename(); - -%if &rule=ISINT %then %do; - &outcol=0; - if not missing(&incol) then do; - &tempcol=input(&incol,?? best32.); - if not missing(&tempcol) then if mod(&tempcol,1)=0 then &outcol=1; - end; - drop &tempcol; -%end; -%else %if &rule=ISNUM %then %do; - /* - credit SØREN LASSEN - https://sasmacro.blogspot.com/2009/06/welcome-isnum-macro.html - */ - &tempcol=input(&incol,?? best32.); - if missing(&tempcol) then &outcol=0; - else &outcol=1; - drop &tempcol; -%end; -%else %if &rule=LIBDS %then %do; - /* match libref.dataset */ - if _n_=1 then do; - retain &tempcol; - &tempcol=prxparse('/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i'); - if missing(&tempcol) then do; - putlog 'ERR' +(-1) "OR: Invalid expression for LIBDS"; - stop; - end; - drop &tempcol; - end; - if prxmatch(&tempcol, trim(&incol)) then &outcol=1; - else &outcol=0; -%end; -%else %if &rule=FORMAT %then %do; - /* match valid format - regex could probably be improved */ - if _n_=1 then do; - retain &tempcol; - &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i'); - if missing(&tempcol) then do; - putlog 'ERR' +(-1) "OR: Invalid expression for FORMAT"; - stop; - end; - drop &tempcol; - end; - if prxmatch(&tempcol, trim(&incol)) then &outcol=1; - else &outcol=0; -%end; - -%mend mp_validatecol; diff --git a/005_macros/ms_adduser2group.sas b/005_macros/ms_adduser2group.sas deleted file mode 100644 index ff09744..0000000 --- a/005_macros/ms_adduser2group.sas +++ /dev/null @@ -1,122 +0,0 @@ -/*** HELP START ***//** - @file - @brief Adds a user to a group on SASjs Server - @details Adds a user to a group based on userid and groupid. Both user and - group must already exist. - - Examples: - - %ms_adduser2group(uid=1,gid=1) - - - @param [in] uid= (0) The User ID to be added - @param [in] gid= (0) The Group ID to contain the new user - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [out] outds= (work.ms_adduser2group) This output dataset will contain - the new list of group members, eg: -|DISPLAYNAME:$18.|USERNAME:$10.|ID:best.| -|---|---|---| -|`Super Admin `|`secretuser `|`1`| -|`Sabir Hassan`|`sabir`|`2`| -|`Mihajlo Medjedovic `|`mihajlo `|`3`| -|`Ivor Townsend `|`ivor `|`4`| -|`New User `|`newuser `|`5`| - - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mp_abort.sas - -

Related Files

- @li ms_creategroup.sas - @li ms_createuser.sas - -**//*** HELP END ***/ - -%macro ms_adduser2group(uid=0 - ,gid=0 - ,outds=work.ms_adduser2group - ,mdebug=0 - ); - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_adduser2group.sas - ,msg=%str(syscc=&syscc on macro entry) -) - -%local fref0 fref1 fref2 libref optval rc msg; -%let fref0=%mf_getuniquefileref(); -%let fref1=%mf_getuniquefileref(); -%let libref=%mf_getuniquelibref(); - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fref0 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then put "accept: application/json"; - put _infile_; -run; - -%if &mdebug=1 %then %do; - %put _local_; - data _null_; - infile &fref0; - input; - put _infile_; - run; -%end; - -proc http method='POST' headerin=&fref0 out=&fref1 - url="&_sasjs_apiserverurl/SASjsApi/group/&gid/&uid"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_adduser2group.sas - ,msg=%str(Issue submitting query to SASjsApi/group) -) - -libname &libref JSON fileref=&fref1; - -data &outds; - set &libref..users; - drop ordinal_root ordinal_users; -%if &mdebug=1 %then %do; - putlog _all_; -%end; -run; - - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_creategroup.sas - ,msg=%str(Issue reading response JSON) -) - -/* reset options */ -options &optval; - -%if &mdebug=0 %then %do; - filename &fref0 clear; - filename &fref1 clear; - libname &libref clear; -%end; -%else %do; - data _null_; - infile &fref1; - input; - putlog _infile_; - run; -%end; - -%mend ms_adduser2group; diff --git a/005_macros/ms_createfile.sas b/005_macros/ms_createfile.sas deleted file mode 100644 index 711284c..0000000 --- a/005_macros/ms_createfile.sas +++ /dev/null @@ -1,112 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a file on SASjs Drive - @details Creates a file on SASjs Drive. To use the file as a Stored Program, - it must have a ".sas" extension. - - Example: - - filename stpcode temp; - data _null_; - file stpcode; - put '%put hello world;'; - run; - %ms_createfile(/some/stored/program.sas, inref=stpcode) - - @param [in] driveloc The full path to the file in SASjs Drive - @param [in] inref= (0) The fileref containing the file to create. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mp_abort.sas - @li ms_deletefile.sas - -**//*** HELP END ***/ - -%macro ms_createfile(driveloc - ,inref=0 - ,mdebug=0 - ); - -/* first, delete in case it exists */ -%ms_deletefile(&driveloc,mdebug=&mdebug) - -%local fname0 fname1 fname2 boundary fname statcd msg optval; -%let fname0=%mf_getuniquefileref(); -%let fname1=%mf_getuniquefileref(); -%let fname2=%mf_getuniquefileref(); -%let boundary=%mf_getuniquename(); - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fname0 termstr=crlf lrecl=32767; - infile &inref end=eof lrecl=32767; - if _n_ = 1 then do; - put "--&boundary."; - put 'Content-Disposition: form-data; name="filePath"'; - put ; - put "&driveloc"; - put "--&boundary"; - put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"'; - put "Content-Type: text/plain"; - put ; - end; - input; - put _infile_; /* add the actual file to be sent */ - if eof then do; - put ; - put "--&boundary--"; - end; -run; - -data _null_; - file &fname1 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary"; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fname0 lrecl=32767; - input; - put _infile_; - data _null_; - infile &fname1 lrecl=32767; - input; - put _infile_; - run; -%end; - -proc http method='POST' in=&fname0 headerin=&fname1 out=&fname2 - url="&_sasjs_apiserverurl/SASjsApi/drive/file"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - -%let statcd=0; -data _null_; - infile &fname2; - input; - putlog _infile_; - if _infile_='{"status":"success"}' then call symputx('statcd',1,'l'); - else call symputx('msg',_infile_,'l'); -run; - -%mp_abort( - iftrue=(&statcd=0) - ,mac=ms_createfile.sas - ,msg=%superq(msg) -) - -/* reset options */ -options &optval; - -%mend ms_createfile; diff --git a/005_macros/ms_creategroup.sas b/005_macros/ms_creategroup.sas deleted file mode 100644 index d78e149..0000000 --- a/005_macros/ms_creategroup.sas +++ /dev/null @@ -1,149 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a group on SASjs Server - @details Creates a group on SASjs Server with the following attributes: - - @li name - @li description - @li isActive - - Examples: - - %ms_creategroup(mynewgroup) - - %ms_creategroup(mynewergroup, desc=The group description) - - @param [in] groupname The group name to create. No spaces or special chars. - @param [in] desc= (0) If no description provided, group name will be used. - @param [in] isactive= (true) Set to false to create an inactive group. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [out] outds= (work.ms_creategroup) This output dataset will contain the - values from the JSON response (such as the id of the new group) -|DESCRIPTION:$1.|GROUPID:best.|ISACTIVE:best.|NAME:$11.| -|---|---|---|---| -|`The group description`|`2 `|`1 `|`mynewergroup `| - - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mp_abort.sas - -

Related Files

- @li ms_creategroup.test.sas - @li ms_getgroups.sas - -**//*** HELP END ***/ - -%macro ms_creategroup(groupname - ,desc=0 - ,isactive=true - ,outds=work.ms_creategroup - ,mdebug=0 - ); - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_creategroup.sas - ,msg=%str(syscc=&syscc on macro entry) -) - -%local fref0 fref1 fref2 libref optval rc msg; -%let fref0=%mf_getuniquefileref(); -%let fref1=%mf_getuniquefileref(); -%let fref2=%mf_getuniquefileref(); -%let libref=%mf_getuniquelibref(); - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fref0 termstr=crlf; - name=quote(cats(symget('groupname'))); - description=quote(cats(symget('desc'))); - if cats(description)='"0"' then description=name; - isactive=symget('isactive'); -%if &mdebug=1 %then %do; - putlog _all_; -%end; - - put '{'@; - put '"name":' name @; - put ',"description":' description @; - put ',"isActive":' isactive @; - put '}'; -run; - -data _null_; - file &fref1 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then do; - put "Content-Type: application/json"; - put "accept: application/json"; - end; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fref0; - input; - put _infile_; - data _null_; - infile &fref1; - input; - put _infile_; - run; -%end; - -proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2 - url="&_sasjs_apiserverurl/SASjsApi/group"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_creategroup.sas - ,msg=%str(Issue submitting query to SASjsApi/group) -) - -libname &libref JSON fileref=&fref2; - -data &outds; - set &libref..root; - drop ordinal_root; -%if &mdebug=1 %then %do; - putlog _all_; -%end; -run; - - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_creategroup.sas - ,msg=%str(Issue reading response JSON) -) - -/* reset options */ -options &optval; - -%if &mdebug=0 %then %do; - filename &fref0 clear; - filename &fref1 clear; - filename &fref2 clear; - libname &libref clear; -%end; -%else %do; - data _null_; - infile &fref2; - input; - putlog _infile_; - run; -%end; - -%mend ms_creategroup; diff --git a/005_macros/ms_createuser.sas b/005_macros/ms_createuser.sas deleted file mode 100644 index 397ed7c..0000000 --- a/005_macros/ms_createuser.sas +++ /dev/null @@ -1,146 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a user on SASjs Server - @details Creates a user on SASjs Server with the following attributes: - - @li UserName - @li Password - @li isAdmin - @li displayName - - The userid is created by sasjs/server. All users are created with `isActive` - set to `true`. - - Example: - - %ms_createuser(newuser,secretpass,displayname=New User!) - - @param [in] username The username to apply. No spaces or special characters. - @param [in] password The initial password to set. - @param [in] isadmin= (false) Set to true to give the user admin rights - @param [in] displayName= (0) Set a friendly name (spaces & special characters - are ok). If not provided, username will be used instead. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [out] outds= (work.ms_createuser) This output dataset will contain the - values from the JSON response (such as the id of the new user) -|ID:best.|DISPLAYNAME:$8.|USERNAME:$8.|ISACTIVE:best.|ISADMIN:best.| -|---|---|---|---|---| -|`6 `|`New User `|`newuser `|`1 `|`0 `| - - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mp_abort.sas - -

Related Files

- @li ms_createuser.test.sas - @li ms_getusers.sas - -**//*** HELP END ***/ - -%macro ms_createuser(username,password - ,isadmin=false - ,displayname=0 - ,outds=work.ms_createuser - ,mdebug=0 - ); - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_createuser.sas - ,msg=%str(syscc=&syscc on macro entry) -) - -%local fref0 fref1 fref2 libref optval rc msg; -%let fref0=%mf_getuniquefileref(); -%let fref1=%mf_getuniquefileref(); -%let fref2=%mf_getuniquefileref(); -%let libref=%mf_getuniquelibref(); - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fref0 termstr=crlf; - username=quote(cats(symget('username'))); - password=quote(cats(symget('password'))); - isadmin=symget('isadmin'); - displayname=quote(cats(symget('displayname'))); - if displayname='"0"' then displayname=username; - -%if &mdebug=1 %then %do; - putlog _all_; -%end; - - put '{'@; - put '"displayName":' displayname @; - put ',"username":' username @; - put ',"password":' password @; - put ',"isAdmin":' isadmin @; - put ',"isActive": true }'; -run; - -data _null_; - file &fref1 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then do; - put "Content-Type: application/json"; - put "accept: application/json"; - end; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fref0; - input; - put _infile_; - data _null_; - infile &fref1; - input; - put _infile_; - run; -%end; - -proc http method='POST' in=&fref0 headerin=&fref1 out=&fref2 - url="&_sasjs_apiserverurl/SASjsApi/user"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_createuser.sas - ,msg=%str(Issue submitting query to SASjsApi/user) -) - -libname &libref JSON fileref=&fref2; - -data &outds; - set &libref..root; - drop ordinal_root; -run; - - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_createuser.sas - ,msg=%str(Issue reading response JSON) -) - -/* reset options */ -options &optval; - -%if &mdebug=1 %then %do; - filename &fref0 clear; - filename &fref1 clear; - filename &fref2 clear; - libname &libref clear; -%end; - -%mend ms_createuser; diff --git a/005_macros/ms_createwebservice.sas b/005_macros/ms_createwebservice.sas deleted file mode 100644 index 04f64e3..0000000 --- a/005_macros/ms_createwebservice.sas +++ /dev/null @@ -1,689 +0,0 @@ -/*** HELP START ***//** - @file ms_createwebservice.sas - @brief Create a Web-Ready Stored Program - @details This macro creates a Stored Program along with the necessary precode - to enable the %webout() macro - - Usage: - - %* compile macros ; - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* parmcards lets us write to a text file from open code ; - filename ft15f001 temp; - parmcards4; - %webout(FETCH) - %* do some sas, any inputs are now already WORK tables; - data example1 example2; - set sashelp.class; - run; - %* send data back; - %webout(OPEN) - %webout(ARR,example1) * Array format, fast, suitable for large tables ; - %webout(OBJ,example2) * Object format, easier to work with ; - %webout(CLOSE) - ;;;; - %ms_createwebservice(path=/Public/app/common,name=appInit,code=ft15f001) - - - - For more examples of using these web services with the SASjs Adapter, see: - https://github.com/sasjs/adapter#readme - - @param [in] path= (0) The full SASjs Drive path in which to create the service - @param [in] name= Stored Program name - @param [in] desc= The description of the service (not implemented yet) - @param [in] precode= Space separated list of filerefs, pointing to the code - that needs to be attached to the beginning of the service (optional) - @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to - be added - @param [in] mDebug= (0) set to 1 to show debug messages in the log - -

SAS Macros

- @li ms_createfile.sas - @li mf_getuser.sas - @li mf_getuniquename.sas - @li mf_getuniquefileref.sas - @li mp_abort.sas - -

Related Files

- @li ms_createwebservice.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro ms_createwebservice(path=0 - ,name=initService - ,precode= - ,code=ft15f001 - ,desc=Not currently used - ,mDebug=0 -)/*/STORE SOURCE*/; - -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -%mp_abort(iftrue=(&syscc ge 4 ) - ,mac=ms_createwebservice - ,msg=%str(syscc=&syscc on macro entry) -) -%mp_abort(iftrue=("&path"="0") - ,mac=ms_createwebservice - ,msg=%str(Path not provided) -) - -/* remove any trailing slash */ -%if "%substr(&path,%length(&path),1)" = "/" %then - %let path=%substr(&path,1,%length(&path)-1); - -/** - * Add webout macro - * These put statements are auto generated - to change the macro, change the - * source (ms_webout) and run `build.py` - */ -%local sasjsref; -%let sasjsref=%mf_getuniquefileref(); -data _null_; - file &sasjsref termstr=crlf lrecl=512; - put "/* Created on %sysfunc(datetime(),datetime19.) by %mf_getuser() */"; -/* WEBOUT BEGIN */ - put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y '; - put ' ,engine=DATASTEP '; - put ' ,missing=NULL '; - put ' ,showmeta=N '; - put ' ,maxobs=MAX '; - put ')/*/STORE SOURCE*/; '; - put '%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval '; - put ' tmpds1 tmpds2 tmpds3 tmpds4; '; - put '%let numcols=0; '; - put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); '; - put ' '; - put '%if &action=OPEN %then %do; '; - put ' options nobomfile; '; - put ' data _null_;file &jref encoding=''utf-8'' lrecl=200; '; - put ' put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; '; - put ' run; '; - put '%end; '; - put '%else %if (&action=ARR or &action=OBJ) %then %do; '; - put ' /* force variable names to always be uppercase in the JSON */ '; - put ' options validvarname=upcase; '; - put ' /* To avoid issues with _webout on EBI - such as encoding diffs and truncation '; - put ' (https://support.sas.com/kb/49/325.html) we use temporary files */ '; - put ' filename _sjs1 temp lrecl=200 ; '; - put ' data _null_; file _sjs1 encoding=''utf-8''; '; - put ' put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; '; - put ' run; '; - put ' /* now write to _webout 1 char at a time */ '; - put ' data _null_; '; - put ' infile _sjs1 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs1 clear; '; - put ' '; - put ' /* grab col defs */ '; - put ' proc contents noprint data=&ds '; - put ' out=_data_(keep=name type length format formatl formatd varnum label); '; - put ' run; '; - put ' %let colinfo=%scan(&syslast,2,.); '; - put ' proc sort data=&colinfo; '; - put ' by varnum; '; - put ' run; '; - put ' /* move meta to mac vars */ '; - put ' data &colinfo; '; - put ' if _n_=1 then call symputx(''numcols'',nobs,''l''); '; - put ' set &colinfo end=last nobs=nobs; '; - put ' name=upcase(name); '; - put ' /* fix formats */ '; - put ' if type=2 or type=6 then do; '; - put ' typelong=''char''; '; - put ' length fmt $49.; '; - put ' if format='''' then fmt=cats(''$'',length,''.''); '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else fmt=cats(format,formatl,''.''); '; - put ' end; '; - put ' else do; '; - put ' typelong=''num''; '; - put ' if format='''' then fmt=''best.''; '; - put ' else if formatl=0 then fmt=cats(format,''.''); '; - put ' else if formatd=0 then fmt=cats(format,formatl,''.''); '; - put ' else fmt=cats(format,formatl,''.'',formatd); '; - put ' end; '; - put ' /* 32 char unique name */ '; - put ' newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); '; - put ' '; - put ' call symputx(cats(''name'',_n_),name,''l''); '; - put ' call symputx(cats(''newname'',_n_),newname,''l''); '; - put ' call symputx(cats(''length'',_n_),length,''l''); '; - put ' call symputx(cats(''fmt'',_n_),fmt,''l''); '; - put ' call symputx(cats(''type'',_n_),type,''l''); '; - put ' call symputx(cats(''typelong'',_n_),typelong,''l''); '; - put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; - put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; - put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; - put ' run; '; - put ' '; - put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql; '; - put ' select count(*) into: lastobs from &ds; '; - put ' %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); '; - put ' '; - put ' %if &engine=PROCJSON %then %do; '; - put ' %if &missing=STRING %then %do; '; - put ' %put &sysmacroname: Special Missings not supported in proc json.; '; - put ' %put &sysmacroname: Switching to DATASTEP engine; '; - put ' %goto datastep; '; - put ' %end; '; - put ' data &tempds; '; - put ' set &ds; '; - put ' &stmt_obs; '; - put ' %if &fmt=N %then format _numeric_ best32.;; '; - put ' /* PRETTY is necessary to avoid line truncation in large files */ '; - put ' filename _sjs2 temp lrecl=131068 encoding=''utf-8''; '; - put ' proc json out=_sjs2 pretty '; - put ' %if &action=ARR %then nokeys ; '; - put ' ;export &tempds / nosastags fmtnumeric; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs2 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs2 clear; '; - put ' %end; '; - put ' %else %if &engine=DATASTEP %then %do; '; - put ' %datastep: '; - put ' %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 '; - put ' %then %do; '; - put ' %put &sysmacroname: &ds NOT FOUND!!!; '; - put ' %return; '; - put ' %end; '; - put ' '; - put ' %if &fmt=Y %then %do; '; - put ' /** '; - put ' * Extract format definitions '; - put ' * First, by getting library locations from dictionary.formats '; - put ' * Then, by exporting the width using proc format '; - put ' * Cannot use maxw from sashelp.vformat as not always populated '; - put ' * Cannot use fmtinfo() as not supported in all flavours '; - put ' */ '; - put ' %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; - put ' proc sql noprint; '; - put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as FMTCAT, '; - put ' FMTNAME '; - put ' from dictionary.formats '; - put ' where fmttype=''F'' and libname is not null '; - put ' and fmtname in (select format from &colinfo where format is not null) '; - put ' order by 1; '; - put ' create table &tmpds2( '; - put ' FMTNAME char(32), '; - put ' LENGTH num '; - put ' ); '; - put ' %local catlist cat fmtlist i; '; - put ' select distinct fmtcat into: catlist separated by '' '' from &tmpds1; '; - put ' %do i=1 %to %sysfunc(countw(&catlist,%str( ))); '; - put ' %let cat=%scan(&catlist,&i,%str( )); '; - put ' proc sql; '; - put ' select distinct fmtname into: fmtlist separated by '' '' '; - put ' from &tmpds1 where fmtcat="&cat"; '; - put ' proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); '; - put ' select &fmtlist; '; - put ' run; '; - put ' proc sql; '; - put ' insert into &tmpds2 select distinct fmtname,length from &tmpds3; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' create table &tmpds4 as '; - put ' select a.*, b.length as MAXW '; - put ' from &colinfo a '; - put ' left join &tmpds2 b '; - put ' on cats(a.format)=cats(upcase(b.fmtname)) '; - put ' order by a.varnum; '; - put ' data _null_; '; - put ' set &tmpds4; '; - put ' if not missing(maxw); '; - put ' call symputx( '; - put ' cats(''fmtlen'',_n_), '; - put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; - put ' ,''l'' '; - put ' ); '; - put ' run; '; - put ' '; - put ' /* configure varlenchk - as we are explicitly shortening the variables */ '; - put ' %let optval=%sysfunc(getoption(varlenchk)); '; - put ' options varlenchk=NOWARN; '; - put ' data _data_(compress=char); '; - put ' /* shorten the new vars */ '; - put ' length '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i $&&fmtlen&i '; - put ' %end; '; - put ' ; '; - put ' /* rename on entry */ '; - put ' set &ds(rename=( '; - put ' %do i=1 %to &numcols; '; - put ' &&name&i=&&newname&i '; - put ' %end; '; - put ' )); '; - put ' &stmt_obs; '; - put ' '; - put ' drop '; - put ' %do i=1 %to &numcols; '; - put ' &&newname&i '; - put ' %end; '; - put ' ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=num %then %do; '; - put ' &&name&i=cats(put(&&newname&i,&&fmt&i)); '; - put ' %end; '; - put ' %else %do; '; - put ' &&name&i=put(&&newname&i,&&fmt&i); '; - put ' %end; '; - put ' %end; '; - put ' if _error_ then do; '; - put ' call symputx(''syscc'',1012); '; - put ' stop; '; - put ' end; '; - put ' run; '; - put ' %let fmtds=&syslast; '; - put ' options varlenchk=&optval; '; - put ' %end; '; - put ' '; - put ' proc format; /* credit yabwon for special null removal */ '; - put ' value bart (default=40) '; - put ' %if &missing=NULL %then %do; '; - put ' ._ - .z = null '; - put ' %end; '; - put ' %else %do; '; - put ' ._ = [quote()] '; - put ' . = null '; - put ' .a - .z = [quote()] '; - put ' %end; '; - put ' other = [best.]; '; - put ' '; - put ' data &tempds; '; - put ' attrib _all_ label=''''; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' length &&name&i $&&fmtlen&i...; '; - put ' format &&name&i $&&fmtlen&i...; '; - put ' %end; '; - put ' %end; '; - put ' %if &fmt=Y %then %do; '; - put ' set &fmtds; '; - put ' %end; '; - put ' %else %do; '; - put ' set &ds; '; - put ' %end; '; - put ' &stmt_obs; '; - put ' format _numeric_ bart.; '; - put ' %do i=1 %to &numcols; '; - put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' &&name&i=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; - put ' %end; '; - put ' %end; '; - put ' run; '; - put ' '; - put ' filename _sjs3 temp lrecl=131068 ; '; - put ' data _null_; '; - put ' file _sjs3 encoding=''utf-8''; '; - put ' if _n_=1 then put "["; '; - put ' set &tempds; '; - put ' if _n_>1 then put "," @; put '; - put ' %if &action=ARR %then "[" ; %else "{" ; '; - put ' %do i=1 %to &numcols; '; - put ' %if &i>1 %then "," ; '; - put ' %if &action=OBJ %then """&&name&i"":" ; '; - put ' "&&name&i"n /* name literal for reserved variable names */ '; - put ' %end; '; - put ' %if &action=ARR %then "]" ; %else "}" ; ; '; - put ' '; - put ' /* close out the table */ '; - put ' data _null_; '; - put ' file _sjs3 mod encoding=''utf-8''; '; - put ' put '']''; '; - put ' run; '; - put ' data _null_; '; - put ' infile _sjs3 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs3 clear; '; - put ' %end; '; - put ' '; - put ' proc sql; '; - put ' drop table &colinfo, &tempds; '; - put ' '; - put ' %if %substr(&showmeta,1,1)=Y %then %do; '; - put ' filename _sjs4 temp lrecl=131068 encoding=''utf-8''; '; - put ' data _null_; '; - put ' file _sjs4; '; - put ' length label $350; '; - put ' put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; '; - put ' do i=1 to &numcols; '; - put ' name=quote(trim(symget(cats(''name'',i)))); '; - put ' format=quote(trim(symget(cats(''fmt'',i)))); '; - put ' label=quote(prxchange(''s/\\/\\\\/'',-1,trim(symget(cats(''label'',i))))); '; - put ' length=quote(trim(symget(cats(''length'',i)))); '; - put ' type=quote(trim(symget(cats(''typelong'',i)))); '; - put ' if i>1 then put "," @@; '; - put ' put name '':{"format":'' format '',"label":'' label '; - put ' '',"length":'' length '',"type":'' type ''}''; '; - put ' end; '; - put ' put ''}}''; '; - put ' run; '; - put ' /* send back to webout */ '; - put ' data _null_; '; - put ' infile _sjs4 lrecl=1 recfm=n; '; - put ' file &jref mod lrecl=1 recfm=n; '; - put ' input sourcechar $char1. @@; '; - put ' format sourcechar hex2.; '; - put ' put sourcechar char1. @@; '; - put ' run; '; - put ' filename _sjs4 clear; '; - put ' %end; '; - put '%end; '; - put ' '; - put '%else %if &action=CLOSE %then %do; '; - put ' data _null_; file &jref encoding=''utf-8'' mod ; '; - put ' put "}"; '; - put ' run; '; - put '%end; '; - put '%mend mp_jsonout; '; - put ' '; - put '%macro mf_getuser( '; - put ')/*/STORE SOURCE*/; '; - put ' %local user; '; - put ' '; - put ' %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; '; - put ' %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; '; - put ' %let user=&SYS_COMPUTE_SESSION_OWNER; '; - put ' %end; '; - put ' %else %if %symexist(_metaperson) %then %do; '; - put ' %if %length(&_metaperson)=0 %then %let user=&sysuserid; '; - put ' /* sometimes SAS will add @domain extension - remove for consistency */ '; - put ' /* but be sure to quote in case of usernames with commas */ '; - put ' %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); '; - put ' %end; '; - put ' %else %let user=&sysuserid; '; - put ' '; - put ' %quote(&user) '; - put ' '; - put '%mend mf_getuser; '; - put ' '; - put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL '; - put ' ,showmeta=N,maxobs=MAX,workobs=0 '; - put '); '; - put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug '; - put ' sasjs_tables; '; - put ' '; - put '%local i tempds; '; - put '%let action=%upcase(&action); '; - put ' '; - put '%if &action=FETCH %then %do; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' options mprint notes mprintnest; '; - put ' %end; '; - put ' %let _webin_file_count=%eval(&_webin_file_count+0); '; - put ' /* now read in the data */ '; - put ' %do i=1 %to &_webin_file_count; '; - put ' %if &_webin_file_count=1 %then %do; '; - put ' %let _webin_fileref1=&_webin_fileref; '; - put ' %let _webin_name1=&_webin_name; '; - put ' %end; '; - put ' data _null_; '; - put ' infile &&_webin_fileref&i termstr=crlf lrecl=32767; '; - put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; - put ' putlog "&&_webin_name&i input statement: " _infile_; '; - put ' stop; '; - put ' data &&_webin_name&i; '; - put ' infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding=''utf-8'' '; - put ' lrecl=32767; '; - put ' input &input_statement; '; - put ' %if %str(&_debug) ge 131 %then %do; '; - put ' if _n_<20 then putlog _infile_; '; - put ' %end; '; - put ' run; '; - put ' %let sasjs_tables=&sasjs_tables &&_webin_name&i; '; - put ' %end; '; - put '%end; '; - put ' '; - put '%else %if &action=OPEN %then %do; '; - put ' /* fix encoding and ensure enough lrecl */ '; - put ' OPTIONS NOBOMFILE lrecl=32767; '; - put ' '; - put ' /* set the header */ '; - put ' %mfs_httpheader(Content-type,application/json) '; - put ' '; - put ' /* setup json. */ '; - put ' data _null_;file &fref encoding=''utf-8'' termstr=lf ; '; - put ' put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; '; - put ' put '',"SYSTIME" : "'' "&SYSTIME" ''"''; '; - put ' run; '; - put ' '; - put '%end; '; - put ' '; - put '%else %if &action=ARR or &action=OBJ %then %do; '; - put ' %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; '; - put ' /* functions in formats unsupported */ '; - put ' %put &sysmacroname: forcing missing back to NULL as feature not supported; '; - put ' %let missing=NULL; '; - put ' %end; '; - put ' %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref '; - put ' ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs '; - put ' ) '; - put '%end; '; - put '%else %if &action=CLOSE %then %do; '; - put ' %if %str(&workobs) > 0 %then %do; '; - put ' /* if debug mode, send back first XX records of each work table also */ '; - put ' data;run;%let tempds=%scan(&syslast,2,.); '; - put ' ods output Members=&tempds; '; - put ' proc datasets library=WORK memtype=data; '; - put ' %local wtcnt;%let wtcnt=0; '; - put ' data _null_; '; - put ' set &tempds; '; - put ' if not (upcase(name) =:"DATA"); /* ignore temp datasets */ '; - put ' if not (upcase(name)=:"_DATA_"); '; - put ' i+1; '; - put ' call symputx(cats(''wt'',i),name,''l''); '; - put ' call symputx(''wtcnt'',i,''l''); '; - put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; - put ' put ",""WORK"":{"; '; - put ' %do i=1 %to &wtcnt; '; - put ' %let wt=&&wt&i; '; - put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; - put ' dsid=open("WORK.&wt",''is''); '; - put ' nlobs=attrn(dsid,''NLOBS''); '; - put ' nvars=attrn(dsid,''NVARS''); '; - put ' rc=close(dsid); '; - put ' if &i>1 then put '',''@; '; - put ' put " ""&wt"" : {"; '; - put ' put ''"nlobs":'' nlobs; '; - put ' put '',"nvars":'' nvars; '; - put ' %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y '; - put ' ,maxobs=&workobs '; - put ' ) '; - put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; - put ' put "}"; '; - put ' %end; '; - put ' data _null_; file &fref mod encoding=''utf-8'' termstr=lf; '; - put ' put "}"; '; - put ' run; '; - put ' %end; '; - put ' /* close off json */ '; - put ' data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; '; - put ' length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; '; - put ' put ",""_DEBUG"" : ""&_debug"" "; '; - put ' _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); '; - put ' put '',"_PROGRAM" : '' _PROGRAM ; '; - put ' autoexec=quote(urlencode(trim(getoption(''autoexec'')))); '; - put ' put '',"AUTOEXEC" : '' autoexec; '; - put ' put ",""MF_GETUSER"" : ""%mf_getuser()"" "; '; - put ' put ",""SYSCC"" : ""&syscc"" "; '; - put ' put ",""SYSENCODING"" : ""&sysencoding"" "; '; - put ' syserrortext=cats(symget(''syserrortext'')); '; - put ' if findc(syserrortext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syserrortext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syserrortext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syserrortext=cats(''"'',syserrortext,''"''); '; - put ' put '',"SYSERRORTEXT" : '' syserrortext; '; - put ' SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); '; - put ' put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; '; - put ' put ",""SYSHOSTNAME"" : ""&syshostname"" "; '; - put ' put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; '; - put ' put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; '; - put ' SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); '; - put ' put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; '; - put ' put ",""SYSJOBID"" : ""&sysjobid"" "; '; - put ' put ",""SYSSCPL"" : ""&sysscpl"" "; '; - put ' put ",""SYSSITE"" : ""&syssite"" "; '; - put ' put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; '; - put ' put ",""SYSUSERID"" : ""&sysuserid"" "; '; - put ' sysvlong=quote(trim(symget(''sysvlong''))); '; - put ' put '',"SYSVLONG" : '' sysvlong; '; - put ' syswarningtext=cats(symget(''syswarningtext'')); '; - put ' if findc(syswarningtext,''"\''!!''0A0D09000E0F010210111A''x) then do; '; - put ' syswarningtext=''"''!!trim( '; - put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; - put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; - put ' prxchange(''s/\x0D/\r/'',-1, /* carriage return */ '; - put ' prxchange(''s/\x09/\\t/'',-1, /* tab */ '; - put ' prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ '; - put ' prxchange(''s/\x0E/\\u000E/'',-1, /* SS */ '; - put ' prxchange(''s/\x0F/\\u000F/'',-1, /* SF */ '; - put ' prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ '; - put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; - put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; - put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; - put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; - put ' prxchange(''s/\\/\\\\/'',-1,syswarningtext) '; - put ' )))))))))))))!!''"''; '; - put ' end; '; - put ' else syswarningtext=cats(''"'',syswarningtext,''"''); '; - put ' put '',"SYSWARNINGTEXT" : '' syswarningtext; '; - put ' put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; '; - put ' length memsize $32; '; - put ' memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; '; - put ' memsize=quote(cats(memsize)); '; - put ' put '',"MEMSIZE" : '' memsize; '; - put ' put "}" @; '; - put ' run; '; - put '%end; '; - put ' '; - put '%mend ms_webout; '; - put ' '; - put '%macro mfs_httpheader(header_name '; - put ' ,header_value '; - put ')/*/STORE SOURCE*/; '; - put '%global sasjs_stpsrv_header_loc; '; - put '%local fref fid i; '; - put ' '; - put '%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do; '; - put ' %put &=fref &=sasjs_stpsrv_header_loc; '; - put ' %put %str(ERR)OR: %sysfunc(sysmsg()); '; - put ' %return; '; - put '%end; '; - put ' '; - put '%let fid=%sysfunc(fopen(&fref,A)); '; - put ' '; - put '%if &fid=0 %then %do; '; - put ' %put %str(ERR)OR: %sysfunc(sysmsg()); '; - put ' %return; '; - put '%end; '; - put ' '; - put '%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value))); '; - put '%let rc=%sysfunc(fwrite(&fid)); '; - put ' '; - put '%let rc=%sysfunc(fclose(&fid)); '; - put '%let rc=%sysfunc(filename(&fref)); '; - put ' '; - put '%mend mfs_httpheader; '; -/* WEBOUT END */ - put '%macro webout(action,ds,dslabel=,fmt=,missing=NULL,showmeta=NO'; - put ' ,maxobs=MAX'; - put ');'; - put ' %ms_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt,missing=&missing'; - put ' ,showmeta=&showmeta,maxobs=&maxobs'; - put ' )'; - put '%mend;'; -run; - -/* add precode and code */ -%local x fref freflist; -%let freflist=&precode &code ; -%do x=1 %to %sysfunc(countw(&freflist)); - %let fref=%scan(&freflist,&x); - %put &sysmacroname: adding &fref; - data _null_; - file &sasjsref lrecl=3000 termstr=crlf mod; - infile &fref lrecl=3000; - input; - put _infile_; - run; -%end; - -/* create the web service */ -%ms_createfile(&path/&name..sas, inref=&sasjsref, mdebug=&mdebug) - -%put ;%put ;%put ;%put ;%put ;%put ; -%put &sysmacroname: STP &name successfully created in &path; -%put ;%put ;%put ; -%put Check it out here:; -%put ;%put ;%put ; -%put &_sasjs_apiserverurl.&_sasjs_apipath?_PROGRAM=&path/&name; -%put ;%put ;%put ;%put ;%put ;%put ; - -%mend ms_createwebservice; diff --git a/005_macros/ms_deletefile.sas b/005_macros/ms_deletefile.sas deleted file mode 100644 index 08d53fd..0000000 --- a/005_macros/ms_deletefile.sas +++ /dev/null @@ -1,48 +0,0 @@ -/*** HELP START ***//** - @file - @brief Deletes a file from SASjs Drive - @details Deletes a file from SASjs Drive, if it exists. - - Example: - - filename stpcode temp; - data _null_; - file stpcode; - put '%put hello world;'; - run; - %ms_createfile(/some/stored/program.sas, inref=stpcode) - - %ms_deletefile(/some/stored/program.sas) - - @param [in] driveloc The full path to the file in SASjs Drive - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

SAS Macros

- @li mf_getuniquefileref.sas - -**//*** HELP END ***/ - -%macro ms_deletefile(driveloc - ,mdebug=0 - ); - -%local headref; -%let headref=%mf_getuniquefileref(); - -data _null_; - file &headref lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - put _infile_; -run; - -proc http method='DELETE' headerin=&headref - url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc"; -%if &mdebug=1 %then %do; - debug level=2; -%end; -run; - -filename &headref clear; - -%mend ms_deletefile; diff --git a/005_macros/ms_getfile.sas b/005_macros/ms_getfile.sas deleted file mode 100644 index 40aab2c..0000000 --- a/005_macros/ms_getfile.sas +++ /dev/null @@ -1,50 +0,0 @@ -/*** HELP START ***//** - @file - @brief Gets a file from SASjs Drive - @details Fetches a file on SASjs Drive and stores it in the output fileref. - - Example: - - %ms_getfile(/Public/app/dc/services/public/settings.sas, outref=myfile) - - @param [in] driveloc The full path to the file in SASjs Drive - @param [out] outref= (msgetfil) The fileref to contain the file. - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - -**//*** HELP END ***/ - -%macro ms_getfile(driveloc - ,outref=msgetfil - ,mdebug=0 - ); - -/* use the recfm in a separate fileref to avoid issues with subsequent reads */ -%local binaryfref floc headref; -%let binaryfref=%mf_getuniquefileref(); -%let headref=%mf_getuniquefileref(); -%let floc=%sysfunc(pathname(work))/%mf_getuniquename().txt; -filename &outref "&floc" lrecl=32767; -filename &binaryfref "&floc" recfm=n; - -data _null_; - file &headref lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - put _infile_; -run; - -proc http method='GET' out=&binaryfref headerin=&headref - url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc"; -%if &mdebug=1 %then %do; - debug level=2; -%end; -run; - -filename &binaryfref clear; -filename &headref clear; - -%mend ms_getfile; diff --git a/005_macros/ms_getgroups.sas b/005_macros/ms_getgroups.sas deleted file mode 100644 index bb54dcb..0000000 --- a/005_macros/ms_getgroups.sas +++ /dev/null @@ -1,147 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fetches the list of groups from SASjs Server - @details Fetches the list of groups from SASjs Server and writes them to an - output dataset. Provide a username to filter for the groups for a particular - user. - - Example: - - %ms_getgroups(outds=userlist) - - With filter on username: - - %ms_getgroups(outds=userlist, user=James) - - With filter on userid: - - %ms_getgroups(outds=userlist, uid=1) - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [in] user= (0) Provide the username on which to filter - @param [in] uid= (0) Provide the userid on which to filter - @param [out] outds= (work.ms_getgroups) This output dataset will contain the - list of groups. Format: -|NAME:$32.|DESCRIPTION:$256.|GROUPID:best.| -|---|---|---| -|`SomeGroup `|`A group `|`1`| -|`Another Group`|`this is a different group`|`2`| -|`admin`|`Administrators `|`3`| - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mp_abort.sas - -

Related Files

- @li ms_creategroup.sas - @li ms_getgroups.test.sas - -**//*** HELP END ***/ - -%macro ms_getgroups( - user=0, - uid=0, - outds=work.ms_getgroups, - mdebug=0 -); - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getgroups.sas - ,msg=%str(syscc=&syscc on macro entry) -) - -%local fref0 fref1 libref optval rc msg url; - -%if %sysget(MODE)=desktop %then %do; - /* groups api does not exist in desktop mode */ - data &outds; - length NAME $32 DESCRIPTION $256. GROUPID 8; - name="&sysuserid"; - description="&sysuserid (group - desktop mode)"; - groupid=1; - output; - stop; - run; - %return; -%end; - -%let fref0=%mf_getuniquefileref(); -%let fref1=%mf_getuniquefileref(); -%let libref=%mf_getuniquelibref(); - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fref0 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then put "accept: application/json"; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fref0; - input; - put _infile_; - run; -%end; - -%if "&user" ne "0" %then %let url=/SASjsApi/user/by/username/&user; -%else %if "&uid" ne "0" %then %let url=/SASjsApi/user/&uid; -%else %let url=/SASjsApi/group; - - -proc http method='GET' headerin=&fref0 out=&fref1 - url="&_sasjs_apiserverurl.&url"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getgroups.sas - ,msg=%str(Issue submitting GET query to SASjsApi) -) - -libname &libref JSON fileref=&fref1; - -%if "&user"="0" and "&uid"="0" %then %do; - data &outds; - length NAME $32 DESCRIPTION $256. GROUPID 8; - if _n_=1 then call missing(of _all_); - set &libref..root; - drop ordinal_root; - run; -%end; -%else %do; - data &outds; - length NAME $32 DESCRIPTION $256. GROUPID 8; - if _n_=1 then call missing(of _all_); - set &libref..groups; - drop ordinal_:; - run; -%end; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getgroups.sas - ,msg=%str(Issue reading response JSON) -) - -/* reset options */ -options &optval; - -%if &mdebug=1 %then %do; - filename &fref0 clear; - filename &fref1 clear; - libname &libref clear; -%end; - -%mend ms_getgroups; diff --git a/005_macros/ms_getusers.sas b/005_macros/ms_getusers.sas deleted file mode 100644 index 23366fa..0000000 --- a/005_macros/ms_getusers.sas +++ /dev/null @@ -1,148 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fetches the list of users from SASjs Server - @details Fetches the list of users from SASjs Server and writes them to an - output dataset. Can also be filtered, for a particular group. - - Example: - - %ms_getusers(outds=userlist) - - Filtering for a group by group name: - - %ms_getusers(outds=work.groupmembers, group=GROUPNAME) - - Filtering for a group by group id: - - %ms_getusers(outds=work.groupmembers, gid=1) - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [in] group= (0) Set to a group name to filter members for that group - @param [in] gid= (0) Set to a group id to filter members for that group - @param [out] outds= (work.ms_getusers) This output dataset will contain the - list of user accounts. Format: -|DISPLAYNAME:$60.|USERNAME:$30.|ID:best.| -|---|---|---| -|`Super Admin `|`secretuser `|`1`| -|`Sabir Hassan`|`sabir`|`2`| -|`Mihajlo Medjedovic `|`mihajlo `|`3`| -|`Ivor Townsend `|`ivor `|`4`| -|`New User `|`newuser `|`5`| - - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquelibref.sas - @li mp_abort.sas - -

Related Files

- @li ms_createuser.sas - @li ms_getgroups.sas - @li ms_getusers.test.sas - -**//*** HELP END ***/ - -%macro ms_getusers( - outds=work.ms_getusers, - group=0, - gid=0, - mdebug=0 -); - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getusers.sas - ,msg=%str(syscc=&syscc on macro entry) -) - -%local fref0 fref1 libref optval rc msg url; -%let fref0=%mf_getuniquefileref(); -%let fref1=%mf_getuniquefileref(); -%let libref=%mf_getuniquelibref(); - -%if %sysget(MODE)=desktop %then %do; - /* users api does not exist in desktop mode */ - data &outds; - length DISPLAYNAME $60 USERNAME:$30 ID 8; - USERNAME="&sysuserid"; - DISPLAYNAME="&sysuserid (desktop mode)"; - ID=1; - output; - stop; - run; - %return; -%end; - -/* avoid sending bom marker to API */ -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -data _null_; - file &fref0 lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then put "accept: application/json"; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &fref0; - input; - put _infile_; - run; -%end; - -%if "&group" ne "0" %then %let url=/SASjsApi/group/by/groupname/&group; -%else %if "&gid" ne "0" %then %let url=/SASjsApi/group/&gid; -%else %let url=/SASjsApi/user; - -proc http method='GET' headerin=&fref0 out=&fref1 - url="&_sasjs_apiserverurl.&url"; -%if &mdebug=1 %then %do; - debug level=1; -%end; -run; - - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getusers.sas - ,msg=%str(Issue submitting API query) -) - -libname &libref JSON fileref=&fref1; - -%if "&group"="0" and "&gid"="0" %then %do; - data &outds; - length DISPLAYNAME $60 USERNAME:$30 ID 8; - if nobs=0 then call missing(of _all_); - set &libref..root nobs=nobs; - drop ordinal_root; - run; -%end; -%else %do; - data &outds; - length DISPLAYNAME $60 USERNAME:$30 ID 8; - if nobs=0 then call missing(of _all_); - set &libref..users nobs=nobs; - drop ordinal_root ordinal_users; - run; -%end; - -%mp_abort( - iftrue=(&syscc ne 0) - ,mac=ms_getusers.sas - ,msg=%str(Issue reading response JSON) -) - -/* reset options */ -options &optval; - -%if &mdebug=1 %then %do; - filename &fref0 clear; - filename &fref1 clear; - libname &libref clear; -%end; - -%mend ms_getusers; diff --git a/005_macros/ms_runstp.sas b/005_macros/ms_runstp.sas deleted file mode 100644 index aed946e..0000000 --- a/005_macros/ms_runstp.sas +++ /dev/null @@ -1,215 +0,0 @@ -/*** HELP START ***//** - @file - @brief Executes a SASjs Server Stored Program - @details Runs a Stored Program (using POST method) and extracts the webout and - log from the response JSON. - - Example: - - %ms_runstp(/some/stored/program - ,debug=131 - ,outref=weboot - ) - - @param [in] pgm The full path to the Stored Program in SASjs Drive (_program - parameter) - @param [in] debug= (131) The value to supply to the _debug URL parameter - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [in] inputparams=(_null_) A dataset containing name/value pairs in the - following format: - |name:$32|value:$10000| - |---|---| - |stpmacname|some value| - |mustbevalidname|can be anything, oops, %abort!!| - @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in - the following format: - |fileref:$8|name:$32|filename:$256| - |---|---|--| - |someref|some_name|some_filename.xls| - |fref2|another_file|zyx_v2.csv| - - @param [out] outref= (outweb) The output fileref to contain the response JSON - (will be created using temp engine) - @param [out] outlogds= (_null_) Set to the name of a dataset to contain the - log. Table format: - |line:$2000| - |---| - |log line 1| - |log line 2| - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mp_abort.sas - @li mp_chop.sas - -**//*** HELP END ***/ - -%macro ms_runstp(pgm - ,debug=131 - ,inputparams=_null_ - ,inputfiles=_null_ - ,outref=outweb - ,outlogds=_null_ - ,mdebug=0 - ); -%local dbg mainref authref boundary; -%let mainref=%mf_getuniquefileref(); -%let authref=%mf_getuniquefileref(); -%let boundary=%mf_getuniquename(); -%if &inputparams=0 %then %let inputparams=_null_; - -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - - -%mp_abort(iftrue=("&pgm"="") - ,mac=&sysmacroname - ,msg=%str(Program not provided) -) - -/* avoid sending bom marker to API */ -%local optval; -%let optval=%sysfunc(getoption(bomfile)); -options nobomfile; - -/* add params */ -data _null_; - file &mainref termstr=crlf lrecl=32767 mod; - length line $1000 name $32 value $32767; - if _n_=1 then call missing(of _all_); - set &inputparams; - put "--&boundary"; - line=cats('Content-Disposition: form-data; name="',name,'"'); - put line; - put ; - put value; -run; - -/* parse input file list */ -%local webcount; -%let webcount=0; -data _null_; - set &inputfiles end=last; - length fileref $8 name $32 filename $256; - call symputx(cats('webref',_n_),fileref,'l'); - call symputx(cats('webname',_n_),name,'l'); - call symputx(cats('webfilename',_n_),filename,'l'); - if last then do; - call symputx('webcount',_n_); - call missing(of _all_); - end; -run; - -/* write out the input files */ -%local i; -%do i=1 %to &webcount; - data _null_; - file &mainref termstr=crlf lrecl=32767 mod; - infile &&webref&i lrecl=32767; - if _n_ = 1 then do; - length line $32767; - line=cats( - 'Content-Disposition: form-data; name="' - ,"&&webname&i" - ,'"; filename="' - ,"&&webfilename&i" - ,'"' - ); - put "--&boundary"; - put line; - put "Content-Type: text/plain"; - put ; - end; - input; - put _infile_; /* add the actual file to be sent */ - run; -%end; - -data _null_; - file &mainref termstr=crlf mod; - put "--&boundary--"; -run; - -data _null_; - file &authref lrecl=1000; - infile "&_sasjs_tokenfile" lrecl=1000; - input; - if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary"; - put _infile_; -run; - -%if &mdebug=1 %then %do; - data _null_; - infile &authref; - input; - put _infile_; - data _null_; - infile &mainref; - input; - put _infile_; - run; -%end; - -%local resp_path; -%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename(); -filename &outref "&resp_path" lrecl=32767; - -/* prepare request*/ -proc http method='POST' headerin=&authref in=&mainref out=&outref - url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131"; -%if &mdebug=1 %then %do; - debug level=2; -%end; -run; - -%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201) -or &mdebug=1 -%then %do; - data _null_;infile &outref;input;putlog _infile_;run; -%end; -%mp_abort( - iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201) - ,mac=&sysmacroname - ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE) -) - -/* reset options */ -options &optval; - -%if &outlogds ne _null_ or &mdebug=1 %then %do; - %local matchstr chopout; - %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784; - %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop); - - %mp_chop("&resp_path" - ,matchvar=matchstr - ,keep=LAST - ,matchpoint=END - ,outfile="&chopout" - ,mdebug=&mdebug - ) - - data &outlogds; - infile "&chopout" lrecl=2000; - length line $2000; - line=_infile_; - %if &mdebug=1 %then %do; - putlog line=; - %end; - run; -%end; - -%if &mdebug=1 %then %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; -%else %do; - /* clear refs */ - filename &authref; - filename &mainref; -%end; -%mend ms_runstp; diff --git a/005_macros/ms_testservice.sas b/005_macros/ms_testservice.sas deleted file mode 100644 index caca8c7..0000000 --- a/005_macros/ms_testservice.sas +++ /dev/null @@ -1,141 +0,0 @@ -/*** HELP START ***//** - @file - @brief Will execute a SASjs web service on SASjs Server - @details Prepares the input files and retrieves the resulting datasets from - the response JSON. - - @param [in] program The Stored Program endpoint to test - @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as - follows: - inputfiles=inref:filename inref2:filename2 - @param [in] inputdatasets= (0) All datasets in this space seperated list are - converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to - the list of `inputfiles` for ingestion. The dataset will be sent with the - same name (no need for a colon modifier). - @param [in] inputparams=(0) A dataset containing name/value pairs in the - following format: - |name:$32|value:$1000| - |---|---| - |stpmacname|some value| - |mustbevalidname|can be anything, oops, %abort!!| - - @param [in] debug= (131) Provide the _debug value to pass to the STP - @param [in] mdebug= (0) Set to 1 to provide macro debugging (this macro) - @param [out] outlib= (0) Output libref to contain the final tables. Set to - 0 if the service output is not in JSON format. - @param [out] outref= (0) Output fileref to create, to contain the full _webout - response. - @param [out] outlogds= (_null_) Set to the name of a dataset to contain the - log. Table format: - |line:$2000| - |---| - |log line 1| - |log line 2| - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mp_abort.sas - @li mp_binarycopy.sas - @li mp_chop.sas - @li mp_ds2csv.sas - @li ms_runstp.sas - -

Related Programs

- @li mp_testservice.test.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro ms_testservice(program, - inputfiles=0, - inputdatasets=0, - inputparams=0, - debug=0, - mdebug=0, - outlib=0, - outref=0, - outlogds=_null_ -)/*/STORE SOURCE*/; -%local dbg i var ds1 fref1 chopout1 chopout2; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -/* convert inputdatasets to filerefs */ -%if "&inputdatasets" ne "0" %then %do; - %if %quote(&inputfiles)=0 %then %let inputfiles=; - %do i=1 %to %sysfunc(countw(&inputdatasets,%str( ))); - %let var=%scan(&inputdatasets,&i,%str( )); - %local dsref&i; - %let dsref&i=%mf_getuniquefileref(); - %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS) - %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.); - %end; -%end; - -/* parse the filerefs - convert to a dataset */ -%let ds1=%mf_getuniquename(); -data &ds1; - length fileref $8 name $32 filename $256 var $300; - if "&inputfiles" ne "0" then do; - webcount=countw("&inputfiles"); - do i=1 to webcount; - var=scan("&inputfiles",i,' '); - fileref=scan(var,1,':'); - name=scan(var,2,':'); - filename=cats(name,'.csv'); - output; - end; - end; -run; - - -/* execute the STP */ -%let fref1=%mf_getuniquefileref(); - -%ms_runstp(&program - ,debug=&debug - ,inputparams=&inputparams - ,inputfiles=&ds1 - ,outref=&fref1 - ,mdebug=&mdebug - ,outlogds=&outlogds -) - - -/* chop out JSON section */ -%local matchstr chopout; -%let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784; -%let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop); - -%mp_chop("%sysfunc(pathname(&fref1,F))" - ,matchvar=matchstr - ,keep=FIRST - ,matchpoint=START - ,offset=-1 - ,outfile="&chopout" - ,mdebug=&mdebug -) - -%if &outlib ne 0 %then %do; - libname &outlib json "&chopout"; -%end; -%if &outref ne 0 %then %do; - filename &outref "&chopout"; -%end; - -%if &mdebug=0 %then %do; - filename &webref clear; - filename &fref1 clear; -%end; -%else %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; - -%mend ms_testservice; diff --git a/005_macros/ms_webout.sas b/005_macros/ms_webout.sas deleted file mode 100644 index aa85895..0000000 --- a/005_macros/ms_webout.sas +++ /dev/null @@ -1,228 +0,0 @@ -/*** HELP START ***//** - @file - @brief Send data to/from sasjs/server - @details This macro should be added to the start of each web service, - **immediately** followed by a call to: - - %ms_webout(FETCH) - - This will read all the input data and create same-named SAS datasets in the - WORK library. You can then insert your code, and send data back using the - following syntax: - - data some datasets; * make some data ; - retain some columns; - run; - - %ms_webout(OPEN) - %ms_webout(ARR,some) * Array format, fast, suitable for large tables ; - %ms_webout(OBJ,datasets) * Object format, easier to work with ; - %ms_webout(CLOSE) - - - @param [in] action Either FETCH, OPEN, ARR, OBJ or CLOSE - @param [in] ds The dataset to send back to the frontend - @param [out] dslabel= value to use instead of table name for sending to JSON - @param [in] fmt= (N) Setting Y converts all vars to their formatted values - @param [out] fref= (_webout) The fileref to which to write the JSON - @param [in] missing= (NULL) Special numeric missing values can be sent as NULL - (eg `null`) or as STRING values (eg `".a"` or `".b"`) - @param [in] showmeta= (N) Set to Y to output metadata alongside each table, - such as the column formats and types. The metadata is contained inside an - object with the same name as the table but prefixed with a dollar sign - ie, - `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` - @param [in] workobs= (0) When set to a positive integer, will create a new - output object (WORK) which contains this number of observations from all - tables in the WORK library. - @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows - that should be converted to output JSON - -

SAS Macros

- @li mf_getuser.sas - @li mp_jsonout.sas - @li mfs_httpheader.sas - -

Related Macros

- @li mv_webout.sas - @li mm_webout.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=N,missing=NULL - ,showmeta=N,maxobs=MAX,workobs=0 -); -%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug - sasjs_tables; - -%local i tempds; -%let action=%upcase(&action); - -%if &action=FETCH %then %do; - %if %str(&_debug) ge 131 %then %do; - options mprint notes mprintnest; - %end; - %let _webin_file_count=%eval(&_webin_file_count+0); - /* now read in the data */ - %do i=1 %to &_webin_file_count; - %if &_webin_file_count=1 %then %do; - %let _webin_fileref1=&_webin_fileref; - %let _webin_name1=&_webin_name; - %end; - data _null_; - infile &&_webin_fileref&i termstr=crlf lrecl=32767; - input; - call symputx('input_statement',_infile_); - putlog "&&_webin_name&i input statement: " _infile_; - stop; - data &&_webin_name&i; - infile &&_webin_fileref&i firstobs=2 dsd termstr=crlf encoding='utf-8' - lrecl=32767; - input &input_statement; - %if %str(&_debug) ge 131 %then %do; - if _n_<20 then putlog _infile_; - %end; - run; - %let sasjs_tables=&sasjs_tables &&_webin_name&i; - %end; -%end; - -%else %if &action=OPEN %then %do; - /* fix encoding and ensure enough lrecl */ - OPTIONS NOBOMFILE lrecl=32767; - - /* set the header */ - %mfs_httpheader(Content-type,application/json) - - /* setup json. */ - data _null_;file &fref encoding='utf-8' termstr=lf ; - put '{"SYSDATE" : "' "&SYSDATE" '"'; - put ',"SYSTIME" : "' "&SYSTIME" '"'; - run; - -%end; - -%else %if &action=ARR or &action=OBJ %then %do; - %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; - /* functions in formats unsupported */ - %put &sysmacroname: forcing missing back to NULL as feature not supported; - %let missing=NULL; - %end; - %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref - ,engine=DATASTEP,missing=&missing,showmeta=&showmeta,maxobs=&maxobs - ) -%end; -%else %if &action=CLOSE %then %do; - %if %str(&workobs) > 0 %then %do; - /* if debug mode, send back first XX records of each work table also */ - data;run;%let tempds=%scan(&syslast,2,.); - ods output Members=&tempds; - proc datasets library=WORK memtype=data; - %local wtcnt;%let wtcnt=0; - data _null_; - set &tempds; - if not (upcase(name) =:"DATA"); /* ignore temp datasets */ - if not (upcase(name)=:"_DATA_"); - i+1; - call symputx(cats('wt',i),name,'l'); - call symputx('wtcnt',i,'l'); - data _null_; file &fref mod encoding='utf-8' termstr=lf; - put ",""WORK"":{"; - %do i=1 %to &wtcnt; - %let wt=&&wt&i; - data _null_; file &fref mod encoding='utf-8' termstr=lf; - dsid=open("WORK.&wt",'is'); - nlobs=attrn(dsid,'NLOBS'); - nvars=attrn(dsid,'NVARS'); - rc=close(dsid); - if &i>1 then put ','@; - put " ""&wt"" : {"; - put '"nlobs":' nlobs; - put ',"nvars":' nvars; - %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y - ,maxobs=&workobs - ) - data _null_; file &fref mod encoding='utf-8' termstr=lf; - put "}"; - %end; - data _null_; file &fref mod encoding='utf-8' termstr=lf; - put "}"; - run; - %end; - /* close off json */ - data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767; - length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; - put ",""_DEBUG"" : ""&_debug"" "; - _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); - put ',"_PROGRAM" : ' _PROGRAM ; - autoexec=quote(urlencode(trim(getoption('autoexec')))); - put ',"AUTOEXEC" : ' autoexec; - put ",""MF_GETUSER"" : ""%mf_getuser()"" "; - put ",""SYSCC"" : ""&syscc"" "; - put ",""SYSENCODING"" : ""&sysencoding"" "; - syserrortext=cats(symget('syserrortext')); - if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syserrortext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syserrortext) - )))))))))))))!!'"'; - end; - else syserrortext=cats('"',syserrortext,'"'); - put ',"SYSERRORTEXT" : ' syserrortext; - SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG'))); - put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG; - put ",""SYSHOSTNAME"" : ""&syshostname"" "; - put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; - put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; - SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); - put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; - put ",""SYSJOBID"" : ""&sysjobid"" "; - put ",""SYSSCPL"" : ""&sysscpl"" "; - put ",""SYSSITE"" : ""&syssite"" "; - put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; - put ",""SYSUSERID"" : ""&sysuserid"" "; - sysvlong=quote(trim(symget('sysvlong'))); - put ',"SYSVLONG" : ' sysvlong; - syswarningtext=cats(symget('syswarningtext')); - if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syswarningtext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syswarningtext) - )))))))))))))!!'"'; - end; - else syswarningtext=cats('"',syswarningtext,'"'); - put ',"SYSWARNINGTEXT" : ' syswarningtext; - put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" '; - length memsize $32; - memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; - memsize=quote(cats(memsize)); - put ',"MEMSIZE" : ' memsize; - put "}" @; - run; -%end; - -%mend ms_webout; diff --git a/006_macros/mcf_getfmttype.sas b/006_macros/mcf_getfmttype.sas deleted file mode 100644 index 782b5ef..0000000 --- a/006_macros/mcf_getfmttype.sas +++ /dev/null @@ -1,118 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the type of the format - @details - Returns the type, eg DATE / DATETIME / TIME (based on hard-coded lookup) - else CHAR / NUM. - - This macro may be extended in the future to support custom formats - this - would necessitate a call to `dosubl()` for running a proc format with cntlout. - - The function itself takes the following (positional) parameters: - - | PARAMETER | DESCRIPTION | - |---|---| - |fmtnm| Format name to be tested. Can be with or without the w.d extension.| - - Usage: - - %mcf_getfmttype(wrap=YES, insert_cmplib=YES) - - data _null_; - fmt1=mcf_getfmttype('DATE9.'); - fmt2=mcf_getfmttype('DATETIME'); - put (fmt:)(=); - run; - %put fmt3=%sysfunc(mcf_getfmttype(TIME9.)); - - Returns: - - > fmt1=DATE fmt2=DATETIME - > fmt3=TIME - - @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper. - @param [out] lib= (work) The output library in which to create the catalog. - @param [out] cat= (sasjs) The output catalog in which to create the package. - @param [out] pkg= (utils) The output package in which to create the function. - Uses a 3 part format: libref.catalog.package - @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and - values inserted only if needed. - -

SAS Macros

- @li mcf_init.sas - -

Related Programs

- @li mcf_getfmttype.test.sas - @li mp_init.sas - - @todo "Custom Format Lookups" To enable site-specific formats, make - use of a set of SASJS_FMTLIST_(DATATYPE) global variables. - -**//*** HELP END ***/ - -%macro mcf_getfmttype(wrap=NO - ,insert_cmplib=DEPRECATED - ,lib=WORK - ,cat=SASJS - ,pkg=UTILS -)/*/STORE SOURCE*/; -%local i var cmpval found; - -%if %mcf_init(mcf_getfmttype)=1 %then %return; - -%if &wrap=YES %then %do; - proc fcmp outlib=&lib..&cat..&pkg; -%end; - -function mcf_getfmttype(fmtnm $) $8; - if substr(fmtnm,1,1)='$' then return('CHAR'); - else do; - /* extract NAME */ - length fmt $32; - fmt=scan(fmtnm,1,'.'); - do while ( - substr(fmt,length(fmt),1) in ('1','2','3','4','5','6','7','8','9','0') - ); - if length(fmt)=1 then fmt='W'; - else fmt=substr(fmt,1,length(fmt)-1); - end; - - /* apply lookups */ - if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT' - ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR' - ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ') - then return('DATETIME'); - else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY' - ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx' - ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB' - ,'MMDDYYC','MMDDYYD','MMDDYYN','MMDDYYP','MMDDYYS','MMDDYYx','MMYY' - ,'MMYYC','MMYYD','MMYYN','MMYYP','MMYYS','MMYYx','MONNAME','MONTH' - ,'MONYY','PDJULG','PDJULI','QTR','QTRR','WEEKDATE','WEEKDATX','WEEKDAY' - ,'WEEKU','WEEKV','WEEKW','WORDDATE','WORDDATX','YEAR','YYMM','YYMMC' - ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS' - ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD' - ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx' - ,'YYQS','YYQx','YYQZ') then return('DATE'); - else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ' - ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ' - ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME'); - else return('NUM'); - end; -endsub; - -%if &wrap=YES %then %do; - quit; -%end; - -/* insert the CMPLIB if not already there */ -%let cmpval=%sysfunc(getoption(cmplib)); -%let found=0; -%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%)))); - %let var=%scan(&cmpval,&i,%str( %(%))); - %if &var=&lib..&cat %then %let found=1; -%end; -%if &found=0 %then %do; - options insert=(CMPLIB=(&lib..&cat)); -%end; - -%mend mcf_getfmttype; diff --git a/006_macros/mcf_init.sas b/006_macros/mcf_init.sas deleted file mode 100644 index d581d5d..0000000 --- a/006_macros/mcf_init.sas +++ /dev/null @@ -1,44 +0,0 @@ -/*** HELP START ***//** - @file - @brief Sets up the mcf_xx functions - @details - There is no (efficient) way to determine if an mcf_xx macro has already been - invoked. So, we make use of a global macro variable list to keep track. - - Usage: - - %mcf_init(MCF_LENGTH) - - Returns: - - > 1 (if already initialised) else 0 - - @param [in] func The function to be initialised - -

Related Macros

- @li mcf_init.test.sas - -**//*** HELP END ***/ - -%macro mcf_init(func -)/*/STORE SOURCE*/; - -%if not (%symexist(SASJS_PREFIX)) %then %do; - %global SASJS_PREFIX; - %let SASJS_PREFIX=SASJS; -%end; - -%let func=%upcase(&func); - -/* the / character is just a seperator */ -%global &sasjs_prefix._FUNCTIONS; -%if %index(&&&sasjs_prefix._FUNCTIONS,&func/)>0 %then %do; - 1 - %return; -%end; -%else %do; - %let &sasjs_prefix._FUNCTIONS=&&&sasjs_prefix._FUNCTIONS &func/; - 0 -%end; - -%mend mcf_init; diff --git a/006_macros/mcf_length.sas b/006_macros/mcf_length.sas deleted file mode 100644 index 5e8ec3d..0000000 --- a/006_macros/mcf_length.sas +++ /dev/null @@ -1,90 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the length of a numeric value - @details - Returns the length, in bytes, of a numeric value. If the value is - missing, then 0 is returned. - - The function itself takes the following (positional) parameters: - - | PARAMETER | DESCRIPTION | - |---|---| - | var | variable (or value) to be tested| - - Usage: - - %mcf_length(wrap=YES, insert_cmplib=YES) - - data _null_; - ina=1; - inb=10000000; - inc=12345678; - ind=.; - outa=mcf_length(ina); - outb=mcf_length(inb); - outc=mcf_length(inc); - outd=mcf_length(ind); - put (out:)(=); - run; - - Returns: - - > outa=3 outb=4 outc=5 outd=0 - - @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper. - @param [out] lib= (work) The output library in which to create the catalog. - @param [out] cat= (sasjs) The output catalog in which to create the package. - @param [out] pkg= (utils) The output package in which to create the function. - Uses a 3 part format: libref.catalog.package - @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and - values inserted only if needed. - -

SAS Macros

- @li mcf_init.sas - -

Related Programs

- @li mcf_length.test.sas - @li mp_init.sas - -**//*** HELP END ***/ - -%macro mcf_length(wrap=NO - ,insert_cmplib=DEPRECATED - ,lib=WORK - ,cat=SASJS - ,pkg=UTILS -)/*/STORE SOURCE*/; -%local i var cmpval found; -%if %mcf_init(mcf_length)=1 %then %return; - -%if &wrap=YES %then %do; - proc fcmp outlib=&lib..&cat..&pkg; -%end; - -function mcf_length(var); - if var=. then len=0; - else if missing(var) or trunc(var,3)=var then len=3; - else if trunc(var,4)=var then len=4; - else if trunc(var,5)=var then len=5; - else if trunc(var,6)=var then len=6; - else if trunc(var,7)=var then len=7; - else len=8; - return(len); -endsub; - -%if &wrap=YES %then %do; - quit; -%end; - -/* insert the CMPLIB if not already there */ -%let cmpval=%sysfunc(getoption(cmplib)); -%let found=0; -%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%)))); - %let var=%scan(&cmpval,&i,%str( %(%))); - %if &var=&lib..&cat %then %let found=1; -%end; -%if &found=0 %then %do; - options insert=(CMPLIB=(&lib..&cat)); -%end; - -%mend mcf_length; diff --git a/006_macros/mcf_string2file.sas b/006_macros/mcf_string2file.sas deleted file mode 100644 index 1e0d2d6..0000000 --- a/006_macros/mcf_string2file.sas +++ /dev/null @@ -1,95 +0,0 @@ -/*** HELP START ***//** - @file - @brief Adds a string to a file - @details Creates an fcmp function for appending a string to an external file. - If the file does not exist, it is created. - - The function itself takes the following (positional) parameters: - - | PARAMETER | DESCRIPTION | - |------------|-------------| - | filepath $ | full path to the file| - | string $ | string to add to the file | - | mode $ | mode of the output - either APPEND (default) or CREATE | - - It returns 0 if successful, or -1 if an error occured. - - Usage: - - %mcf_string2file(wrap=YES, insert_cmplib=YES) - - data _null_; - rc=mcf_string2file( - "%sysfunc(pathname(work))/newfile.txt" - , "This is a test" - , "CREATE"); - run; - - data _null_; - infile "%sysfunc(pathname(work))/newfile.txt"; - input; - putlog _infile_; - run; - - @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper. - @param [out] lib= (work) The output library in which to create the catalog. - @param [out] cat= (sasjs) The output catalog in which to create the package. - @param [out] pkg= (utils) The output package in which to create the function. - Uses a 3 part format: libref.catalog.package - @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and - values inserted only if needed. - -

SAS Macros

- @li mcf_init.sas - -

Related Programs

- @li mcf_stpsrv_header.test.sas - @li mp_init.sas - -**//*** HELP END ***/ - -%macro mcf_string2file(wrap=NO - ,insert_cmplib=DEPRECATED - ,lib=WORK - ,cat=SASJS - ,pkg=UTILS -)/*/STORE SOURCE*/; -%local i var cmpval found; -%if %mcf_init(mcf_string2file)=1 %then %return; - -%if &wrap=YES %then %do; - proc fcmp outlib=&lib..&cat..&pkg; -%end; - -function mcf_string2file(filepath $, string $, mode $); - if mode='APPEND' then fmode='a'; - else fmode='o'; - length fref $8; - rc=filename(fref,filepath); - if (rc ne 0) then return( -1 ); - fid = fopen(fref,fmode); - if (fid = 0) then return( -1 ); - rc=fput(fid, string); - rc=fwrite(fid); - rc=fclose(fid); - rc=filename(fref); - return(0); -endsub; - - -%if &wrap=YES %then %do; - quit; -%end; - -/* insert the CMPLIB if not already there */ -%let cmpval=%sysfunc(getoption(cmplib)); -%let found=0; -%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%)))); - %let var=%scan(&cmpval,&i,%str( %(%))); - %if &var=&lib..&cat %then %let found=1; -%end; -%if &found=0 %then %do; - options insert=(CMPLIB=(&lib..&cat)); -%end; - -%mend mcf_string2file; diff --git a/006_macros/mddl_dc_difftable.sas b/006_macros/mddl_dc_difftable.sas deleted file mode 100644 index 4df0222..0000000 --- a/006_macros/mddl_dc_difftable.sas +++ /dev/null @@ -1,44 +0,0 @@ -/*** HELP START ***//** - @file - @brief Difftable DDL - @details Used to store changes to tables. Used by mp_storediffs.sas - and mp_stackdiffs.sas - -**//*** HELP END ***/ - - -%macro mddl_dc_difftable(libds=WORK.DIFFTABLE); - - proc sql; - create table &libds( - load_ref char(36) label='unique load reference', - processed_dttm num format=E8601DT26.6 label='Processed at timestamp', - libref char(8) label='Library Reference (8 chars)', - dsn char(32) label='Dataset Name (32 chars)', - key_hash char(32) label= - 'MD5 Hash of primary key values (pipe seperated)', - move_type char(1) label='Either (A)ppended, (D)eleted or (M)odified', - is_pk num label='Is Primary Key Field? (1/0)', - is_diff num label= - 'Did value change? (1/0/-1). Always -1 for appends and deletes.', - tgtvar_type char(1) label='Either (C)haracter or (N)umeric', - tgtvar_nm char(32) label='Target variable name (32 chars)', - oldval_num num format=best32. label='Old (numeric) value', - newval_num num format=best32. label='New (numeric) value', - oldval_char char(32765) label='Old (character) value', - newval_char char(32765) label='New (character) value' - ); - - %local lib; - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create - pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm) - /nomiss unique; - quit; - -%mend mddl_dc_difftable; diff --git a/006_macros/mddl_dc_filterdetail.sas b/006_macros/mddl_dc_filterdetail.sas deleted file mode 100644 index 3148f01..0000000 --- a/006_macros/mddl_dc_filterdetail.sas +++ /dev/null @@ -1,40 +0,0 @@ -/*** HELP START ***//** - @file - @brief Filtertable DDL - @details For storing detailed filter values. Used by - mp_filterstore.sas. - -**//*** HELP END ***/ - - -%macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL); - -%local nn lib; -%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - %let nn=not null; -%end; -%else %let nn=; - - proc sql; - create table &libds( - filter_hash char(32) &nn, - filter_line num &nn, - group_logic char(3) &nn, - subgroup_logic char(3) &nn, - subgroup_id num &nn, - variable_nm varchar(32) &nn, - operator_nm varchar(12) &nn, - raw_value varchar(4000) &nn, - processed_dttm num &nn format=E8601DT26.6 - ); - - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique; - quit; - -%mend mddl_dc_filterdetail; diff --git a/006_macros/mddl_dc_filtersummary.sas b/006_macros/mddl_dc_filtersummary.sas deleted file mode 100644 index e3858e7..0000000 --- a/006_macros/mddl_dc_filtersummary.sas +++ /dev/null @@ -1,35 +0,0 @@ -/*** HELP START ***//** - @file - @brief Filtersummary DDL - @details For storing summary filter values. Used by - mp_filterstore.sas. - -**//*** HELP END ***/ - - -%macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY); - -%local nn lib; -%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - %let nn=not null; -%end; -%else %let nn=; - - proc sql; - create table &libds( - filter_rk num &nn, - filter_hash char(32) &nn, - filter_table char(41) &nn, - processed_dttm num &nn format=E8601DT26.6 - ); - - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create filter_rk /nomiss unique; - quit; - -%mend mddl_dc_filtersummary; diff --git a/006_macros/mddl_dc_locktable.sas b/006_macros/mddl_dc_locktable.sas deleted file mode 100644 index dd42340..0000000 --- a/006_macros/mddl_dc_locktable.sas +++ /dev/null @@ -1,41 +0,0 @@ -/*** HELP START ***//** - @file - @brief Locktable DDL - @details For "locking" tables prior to multipass loads. Used by - mp_lockanytable.sas - -**//*** HELP END ***/ - - -%macro mddl_dc_locktable(libds=WORK.LOCKTABLE); - -%local nn lib; -%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - %let nn=not null; -%end; -%else %let nn=; - - proc sql; - create table &libds( - lock_lib char(8), - lock_ds char(32), - lock_status_cd char(10) &nn, - lock_user_nm char(100) &nn , - lock_ref char(200), - lock_pid char(10), - lock_start_dttm num format=E8601DT26.6, - lock_end_dttm num format=E8601DT26.6 - ); - - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create - pk_mp_lockanytable=(lock_lib lock_ds) - /nomiss unique; - quit; - -%mend mddl_dc_locktable; diff --git a/006_macros/mddl_dc_maxkeytable.sas b/006_macros/mddl_dc_maxkeytable.sas deleted file mode 100644 index 07e634e..0000000 --- a/006_macros/mddl_dc_maxkeytable.sas +++ /dev/null @@ -1,33 +0,0 @@ -/*** HELP START ***//** - @file - @brief Maxkeytable DDL - @details For storing the maximum retained key information. Used - by mp_retainedkey.sas - -**//*** HELP END ***/ - - -%macro mddl_dc_maxkeytable(libds=WORK.MAXKEYTABLE); - - proc sql; - create table &libds( - keytable varchar(41) label='Base table in libref.dataset format', - keycolumn char(32) format=$32. - label='The Retained key field containing the key values.', - max_key num label= - 'Integer representing current max RK or SK value in the KEYTABLE', - processed_dttm num format=E8601DT26.6 - label='Datetime this value was last updated' - ); - - %local lib; - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create keytable /nomiss unique; - quit; - -%mend mddl_dc_maxkeytable; diff --git a/006_macros/mddl_sas_cntlout.sas b/006_macros/mddl_sas_cntlout.sas deleted file mode 100644 index e0164b7..0000000 --- a/006_macros/mddl_sas_cntlout.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief The CNTLOUT table generated by proc format - @details The actual CNTLOUT table may have varying variable lengths, - depending on the data values, therefore the max possible lengths - (given various practical restrictions) are described here to enable - consistency when dealing with format data. - -**//*** HELP END ***/ - - -%macro mddl_sas_cntlout(libds=WORK.CNTLOUT); - - proc sql; - create table &libds( - TYPE char(1) label='Type of format - either N (num fmt), C (char fmt), I (num infmt) or J (char infmt)' - ,FMTNAME char(32) label='Format name' - ,FMTROW num label='CALCULATED Position of record by FMTNAME (reqd for multilabel formats)' - ,START char(32767) label='Starting value for format' - /* - Keep lengths of START and END the same to avoid this err: - "Start is greater than end: -<." - Similar usage note: https://support.sas.com/kb/69/330.html - */ - ,END char(32767) label='Ending value for format' - ,LABEL char(32767) label='Format value label' - ,MIN num length=3 label='Minimum length' - ,MAX num length=3 label='Maximum length' - ,DEFAULT num length=3 label='Default length' - ,LENGTH num length=3 label='Format length' - ,FUZZ num label='Fuzz value' - ,PREFIX char(2) label='Prefix characters' - ,MULT num label='Multiplier' - ,FILL char(1) label='Fill character' - ,NOEDIT num length=3 label='Is picture string noedit?' - ,SEXCL char(1) label='Start exclusion' - ,EEXCL char(1) label='End exclusion' - ,HLO char(13) label='Additional information. M=MultiLabel' - ,DECSEP char(1) label='Decimal separator' - ,DIG3SEP char(1) label='Three-digit separator' - ,DATATYPE char(8) label='Date/time/datetime?' - ,LANGUAGE char(8) label='Language for date strings' - ); - - %local lib; - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - - proc datasets lib=&lib noprint; - modify %scan(&libds,-1,.); - index create - pk_cntlout=(type fmtname fmtrow) - /nomiss unique; - quit; - -%mend mddl_sas_cntlout; diff --git a/004_macros/mmx_createmetafolder.sas b/006_macros/mmx_createmetafolder.sas similarity index 100% rename from 004_macros/mmx_createmetafolder.sas rename to 006_macros/mmx_createmetafolder.sas diff --git a/004_macros/mmx_deletemetafolder.sas b/006_macros/mmx_deletemetafolder.sas similarity index 100% rename from 004_macros/mmx_deletemetafolder.sas rename to 006_macros/mmx_deletemetafolder.sas diff --git a/004_macros/mmx_spkexport.sas b/006_macros/mmx_spkexport.sas similarity index 100% rename from 004_macros/mmx_spkexport.sas rename to 006_macros/mmx_spkexport.sas diff --git a/007_macros/mx_createwebservice.sas b/007_macros/mx_createwebservice.sas deleted file mode 100644 index 007f22e..0000000 --- a/007_macros/mx_createwebservice.sas +++ /dev/null @@ -1,98 +0,0 @@ -/*** HELP START ***//** - @file mx_createwebservice.sas - @brief Create a web service in SAS 9, Viya or SASjs - @details Creates a SASJS ready Stored Process in SAS 9, a Job Execution - Service in SAS Viya, or a Stored Program on SASjs Server - depending on the - executing environment. - -Usage: - - %* compile macros ; - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* write some code; - filename ft15f001 temp; - parmcards4; - %* fetch any data from frontend ; - %webout(FETCH) - data example1 example2; - set sashelp.class; - run; - %* send data back; - %webout(OPEN) - %webout(ARR,example1) * Array format, fast, suitable for large tables ; - %webout(OBJ,example2) * Object format, easier to work with ; - %webout(CLOSE) - ;;;; - - %* create the service (including webout macro and dependencies); - %mx_createwebservice(path=/Public/app/common,name=appInit,replace=YES) - -

SAS Macros

- @li mf_getplatform.sas - @li mm_createwebservice.sas - @li ms_createwebservice.sas - @li mv_createwebservice.sas - - @param [in,out] path= The full folder path where the service will be created - @param [in,out] name= Service name. Avoid spaces. - @param [in] desc= The description of the service (optional) - @param [in] precode= Space separated list of filerefs, pointing to the code - that needs to be attached to the beginning of the service (optional) - @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to - be added - @param [in] replace= (YES) Select YES to replace any existing service in that - location - @param [in] mDebug= (0) set to 1 to show debug messages in the log - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mx_createwebservice(path=HOME - ,name=initService - ,precode= - ,code=ft15f001 - ,desc=This service was created by the mp_createwebservice macro - ,replace=YES - ,mdebug=0 -)/*/STORE SOURCE*/; - -%if &syscc ge 4 %then %do; - %put syscc=&syscc - &sysmacroname will not execute in this state; - %return; -%end; - -%local platform; %let platform=%mf_getplatform(); -%if &platform=SASVIYA %then %do; - %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder; - %mv_createwebservice(path=&path - ,name=&name - ,code=&code - ,precode=&precode - ,desc=&desc - ,replace=&replace - ) -%end; -%else %if &platform=SASJS %then %do; - %if "&path"="HOME" %then %let path=/Users/&_sasjs_username/My Folder; - %ms_createwebservice(path=&path - ,name=&name - ,code=&code - ,precode=&precode - ,mdebug=&mdebug - ) -%end; -%else %do; - %if "&path"="HOME" %then %let path=/User Folders/&_METAPERSON/My Folder; - %mm_createwebservice(path=&path - ,name=&name - ,code=&code - ,precode=&precode - ,desc=&desc - ,replace=&replace - ) -%end; - -%mend mx_createwebservice; diff --git a/007_macros/mx_getcode.sas b/007_macros/mx_getcode.sas deleted file mode 100644 index f035c85..0000000 --- a/007_macros/mx_getcode.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fetches code from Viya Job, SAS 9 STP, or SASjs Server STP - @details When building applications that run on multiple flavours of SAS, it - is convenient to use a single macro (like this one) to fetch the source - code from a Viya Job, SAS 9 Stored Process, or SASjs Stored Program. - - The alternative would be to compile a generic macro in target-specific - folders (SASVIYA, SAS9 and SASJS). This avoids compiling unnecessary macros - at the expense of a more complex sasjsconfig.json setup. - - - @param [in] loc The full path to the Viya Job, SAS 9 Stored Process or SASjs - Stored Program in Drive or Metadata, WITHOUT the .sas extension (SASjs only) - @param [out] outref= (0) The fileref to create, which will contain the source - code. - -

SAS Macros

- @li mf_getplatform.sas - @li mm_getstpcode.sas - @li ms_getfile.sas - @li mv_getjobcode.sas - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mx_getcode(loc,outref=0 -)/*/STORE SOURCE*/; - -%local platform name shortloc; -%let platform=%mf_getplatform(); - -%if &platform=SASJS %then %do; - %ms_getfile(&loc..sas, outref=&outref) -%end; -%else %if &platform=SAS9 or &platform=SASMETA %then %do; - %mm_getstpcode(tree=&loc,outref=&outref) -%end; -%else %if &platform=SASVIYA %then %do; - /* extract name & path from &loc */ - data _null_; - loc=symget('loc'); - name=scan(loc,-1,'/'); - shortloc=substr(loc,1,length(loc)-length(name)-1); - call symputx('name',name,'l'); - call symputx('shortloc',shortloc,'l'); - run; - %mv_getjobcode( - path=&shortloc, - name=&name, - outref=&outref - ) -%end; -%else %put &sysmacroname: &platform is unsupported!!!; - -%mend mx_getcode; diff --git a/007_macros/mx_testservice.sas b/007_macros/mx_testservice.sas deleted file mode 100644 index 7234791..0000000 --- a/007_macros/mx_testservice.sas +++ /dev/null @@ -1,299 +0,0 @@ -/*** HELP START ***//** - @file - @brief Will execute a SASjs web service on SAS 9, Viya or SASjs Server - @details Prepares the input files and retrieves the resulting datasets from - the response JSON. - - Note - the _webout fileref should NOT be assigned prior to running this macro. - - @param [in] program The _PROGRAM endpoint to test - @param [in] inputfiles=(0) A list of space seperated fileref:filename pairs as - follows: - inputfiles=inref:filename inref2:filename2 - @param [in] inputdatasets= (0) All datasets in this space seperated list are - converted into SASJS-formatted CSVs (see mp_ds2csv.sas) files and added to - the list of `inputfiles` for ingestion. The dataset will be sent with the - same name (no need for a colon modifier). - @param [in] inputparams=(0) A dataset containing name/value pairs in the - following format: - |name:$32|value:$1000| - |---|---| - |stpmacname|some value| - |mustbevalidname|can be anything, oops, %abort!!| - - @param [in] debug= (log) Provide the _debug value - @param [in] mdebug= (0) Set to 1 to provide macro debugging - @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return. For - more info, see mv_getjobresult.sas - @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute - context on which to run the service - @param [out] outlib= (0) Output libref to contain the final tables. Set to - 0 if the service output is not in JSON format. - @param [out] outref= (0) Output fileref to create, to contain the full _webout - response. - -

SAS Macros

- @li mf_getplatform.sas - @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mp_abort.sas - @li mp_binarycopy.sas - @li mp_chop.sas - @li mp_ds2csv.sas - @li ms_testservice.sas - @li mv_getjobresult.sas - @li mv_jobflow.sas - -

Related Programs

- @li mx_testservice.test.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mx_testservice(program, - inputfiles=0, - inputdatasets=0, - inputparams=0, - debug=log, - mdebug=0, - outlib=0, - outref=0, - viyaresult=WEBOUT_JSON, - viyacontext=SAS Job Execution compute context -); -%local dbg pcnt fref1 fref2 webref webrefpath i webcount var platform; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -/* sanitise inputparams */ -%let pcnt=0; -%if &inputparams ne 0 %then %do; - data _null_; - set &inputparams; - if not nvalid(name,'v7') then putlog (_all_)(=); - else if name in ( - 'program','inputfiles','inputparams','debug','outlib','outref' - ) then putlog (_all_)(=); - else do; - x+1; - call symputx(name,quote(cats(value)),'l'); - call symputx(cats('pval',x),name,'l'); - call symputx('pcnt',x,'l'); - end; - run; - %mp_abort(iftrue= (%mf_nobs(&inputparams) ne &pcnt) - ,mac=&sysmacroname - ,msg=%str(Invalid values in &inputparams) - ) -%end; - -/* convert inputdatasets to filerefs */ -%if "&inputdatasets" ne "0" %then %do; - %if %quote(&inputfiles)=0 %then %let inputfiles=; - %do i=1 %to %sysfunc(countw(&inputdatasets,%str( ))); - %let var=%scan(&inputdatasets,&i,%str( )); - %local dsref&i; - %let dsref&i=%mf_getuniquefileref(); - %mp_ds2csv(&var,outref=&&dsref&i,headerformat=SASJS) - %let inputfiles=&inputfiles &&dsref&i:%scan(&var,-1,.); - %end; -%end; - -%let platform=%mf_getplatform(); -%let fref1=%mf_getuniquefileref(); -%let fref2=%mf_getuniquefileref(); -%let webref=%mf_getuniquefileref(); -%let webrefpath=%sysfunc(pathname(work))/%mf_getuniquename(); -/* mp_chop requires a physical path as input */ -filename &webref "&webrefpath"; - -%if &platform=SASMETA %then %do; - - /* parse the input files */ - %if %quote(&inputfiles) ne 0 %then %do; - %let webcount=%sysfunc(countw(&inputfiles)); - %put &=webcount; - %do i=1 %to &webcount; - %let var=%scan(&inputfiles,&i,%str( )); - %local webfref&i webname&i; - %let webref&i=%scan(&var,1,%str(:)); - %let webname&i=%scan(&var,2,%str(:)); - %put webref&i=&&webref&i; - %put webname&i=&&webname&i; - %end; - %end; - %else %let webcount=0; - - proc stp program="&program"; - inputparam _program="&program" - %do i=1 %to &webcount; - %if &webcount=1 %then %do; - _webin_fileref="&&webref&i" - _webin_name="&&webname&i" - %end; - %else %do; - _webin_fileref&i="&&webref&i" - _webin_name&i="&&webname&i" - %end; - %end; - _webin_file_count="&webcount" - _debug="&debug" - %do i=1 %to &pcnt; - /* resolve name only, proc stp fetches value */ - &&pval&i=&&&&&&pval&i - %end; - ; - %do i=1 %to &webcount; - inputfile &&webref&i; - %end; - outputfile _webout=&webref; - run; - - data _null_; - infile &webref; - file &fref1; - input; - length line $10000; - if index(_infile_,'>>weboutBEGIN<<') then do; - line=tranwrd(_infile_,'>>weboutBEGIN<<',''); - put line; - end; - else if index(_infile_,'>>weboutEND<<') then do; - line=tranwrd(_infile_,'>>weboutEND<<',''); - put line; - stop; - end; - else put _infile_; - run; - data _null_; - infile &fref1; - input; - put _infile_; - run; - %if &outlib ne 0 %then %do; - libname &outlib json (&fref1); - %end; - %if &outref ne 0 %then %do; - filename &outref temp; - %mp_binarycopy(inref=&webref,outref=&outref) - %end; - -%end; -%else %if &platform=SASVIYA %then %do; - - /* prepare inputparams */ - %local ds1; - %let ds1=%mf_getuniquename(); - %if "&inputparams" ne "0" %then %do; - proc transpose data=&inputparams out=&ds1; - id name; - var value; - run; - %end; - %else %do; - data &ds1;run; - %end; - - /* parse the input files - convert to sasjs params */ - %local webcount i var sasjs_tables; - %if %quote(&inputfiles) ne 0 %then %do; - %let webcount=%sysfunc(countw(&inputfiles)); - %put &=webcount; - %do i=1 %to &webcount; - %let var=%scan(&inputfiles,&i,%str( )); - %local webfref&i webname&i sasjs&i.data; - %let webref&i=%scan(&var,1,%str(:)); - %let webname&i=%scan(&var,2,%str(:)); - %put webref&i=&&webref&i; - %put webname&i=&&webname&i; - - %let sasjs_tables=&sasjs_tables &&webname&i; - data _null_; - infile &&webref&i lrecl=32767; - input; - if _n_=1 then call symputx("sasjs&i.data",_infile_); - else call symputx( - "sasjs&i.data",cats(symget("sasjs&i.data"),'0D0A'x,_infile_) - ); - putlog "&sysmacroname infile: " _infile_; - run; - data &ds1; - set &ds1; - length sasjs&i.data $32767 sasjs_tables $1000; - sasjs&i.data=symget("sasjs&i.data"); - sasjs_tables=symget("sasjs_tables"); - run; - %end; - %end; - %else %let webcount=0; - - data &ds1; - retain _program "&program"; - retain _contextname "&viyacontext"; - set &ds1; - putlog "&sysmacroname inputparams:"; - putlog (_all_)(=); - run; - - %mv_jobflow(inds=&ds1 - ,maxconcurrency=1 - ,outds=work.results - ,outref=&fref1 - ,mdebug=&mdebug - ) - /* show the log */ - data _null_; - infile &fref1; - input; - putlog _infile_; - run; - /* get the uri to fetch results */ - data _null_; - set work.results; - call symputx('uri',uri); - putlog "&sysmacroname: fetching results for " uri; - run; - /* fetch results from webout.json */ - %mv_getjobresult(uri=&uri, - result=&viyaresult, - outref=&outref, - outlib=&outlib, - mdebug=&mdebug - ) - -%end; -%else %if &platform=SASJS %then %do; - - %ms_testservice(&program - ,inputfiles=&inputfiles - ,inputdatasets=&inputdatasets - ,inputparams=&inputparams - ,debug=&debug - ,mdebug=&mdebug - ,outlib=&outlib - ,outref=&outref - ) - -%end; -%else %do; - %put %str(ERR)OR: Unrecognised platform: &platform; -%end; - -%if &mdebug=0 %then %do; - filename &fref1 clear; - %if &platform ne SASJS %then %do; - filename &fref2 clear; - filename &webref clear; - %end; -%end; -%else %do; - %put &sysmacroname exit vars:; - %put _local_; -%end; - -%mend mx_testservice; diff --git a/001_macros/mfv_existfile.sas b/008_macros/mfv_existfile.sas similarity index 100% rename from 001_macros/mfv_existfile.sas rename to 008_macros/mfv_existfile.sas diff --git a/001_macros/mfv_existfolder.sas b/008_macros/mfv_existfolder.sas similarity index 100% rename from 001_macros/mfv_existfolder.sas rename to 008_macros/mfv_existfolder.sas diff --git a/001_macros/mfv_existsashdat.sas b/008_macros/mfv_existsashdat.sas similarity index 100% rename from 001_macros/mfv_existsashdat.sas rename to 008_macros/mfv_existsashdat.sas diff --git a/008_macros/ml_gsubfile.sas b/008_macros/ml_gsubfile.sas deleted file mode 100644 index b4bc904..0000000 --- a/008_macros/ml_gsubfile.sas +++ /dev/null @@ -1,52 +0,0 @@ -/*** HELP START ***//** - @file ml_gsubfile.sas - @brief Compiles the gsubfile.lua lua file - @details Writes gsubfile.lua to the work directory - and then includes it. - Usage: - - %ml_gsubfile() - -**//*** HELP END ***/ - -%macro ml_gsubfile(); -data _null_; - file "%sysfunc(pathname(work))/ml_gsubfile.lua"; - put 'local fpath, outpath, file, fcontent '; - put ' '; - put '-- configure in / out paths '; - put 'fpath = sas.symget("file") '; - put 'outpath = sas.symget("outfile") '; - put 'if ( outpath == 0 ) '; - put 'then '; - put ' outpath=fpath '; - put 'end '; - put ' '; - put '-- open file and perform the substitution '; - put 'file = io.open(fpath,"r") '; - put 'fcontent = file:read("*all") '; - put 'file:close() '; - put 'fcontent = string.gsub( '; - put ' fcontent, '; - put ' sas.symget(sas.symget("patternvar")), '; - put ' sas.symget(sas.symget("replacevar")) '; - put ') '; - put ' '; - put '-- write the file back out '; - put 'file = io.open(outpath, "w+") '; - put 'io.output(file) '; - put 'io.write(fcontent) '; - put 'io.close(file) '; -run; - -/* ensure big enough lrecl to avoid lua compilation issues */ -%local optval; -%let optval=%sysfunc(getoption(lrecl)); -options lrecl=1024; - -/* execute the lua code by using a .lua extension */ -%inc "%sysfunc(pathname(work))/ml_gsubfile.lua" /source2; - -options lrecl=&optval; - -%mend ml_gsubfile; diff --git a/008_macros/ml_json.sas b/008_macros/ml_json.sas deleted file mode 100644 index 005d1dd..0000000 --- a/008_macros/ml_json.sas +++ /dev/null @@ -1,402 +0,0 @@ -/*** HELP START ***//** - @file ml_json.sas - @brief Compiles the json.lua lua file - @details Writes json.lua to the work directory - and then includes it. - Usage: - - %ml_json() - -**//*** HELP END ***/ - -%macro ml_json(); -data _null_; - file "%sysfunc(pathname(work))/ml_json.lua"; - put '-- NOTE - THE COPYRIGHT BELOW IS IN RELATION TO THE JSON.LUA FILE ONLY '; - put '-- THIS FILE STARTS ON THE NEXT LINE AND WILL FINISH WITH "JSON.LUA ENDS HERE" '; - put '-- '; - put '-- json.lua '; - put '-- '; - put '-- Copyright (c) 2019 rxi '; - put '-- '; - put '-- Permission is hereby granted, free of charge, to any person obtaining a copy of '; - put '-- this software and associated documentation files (the "Software"), to deal in '; - put '-- the Software without restriction, including without limitation the rights to '; - put '-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies '; - put '-- of the Software, and to permit persons to whom the Software is furnished to do '; - put '-- so, subject to the following conditions: '; - put '-- '; - put '-- The above copyright notice and this permission notice shall be included in all '; - put '-- copies or substantial portions of the Software. '; - put '-- '; - put '-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR '; - put '-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, '; - put '-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE '; - put '-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER '; - put '-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, '; - put '-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE '; - put '-- SOFTWARE. '; - put '-- '; - put ' '; - put 'json = { _version = "0.1.2" } '; - put ' '; - put '------------------------------------------------------------------------------- '; - put '-- Encode '; - put '------------------------------------------------------------------------------- '; - put ' '; - put 'local encode '; - put ' '; - put 'local escape_char_map = { '; - put ' [ "\\" ] = "\\\\", '; - put ' [ "\"" ] = "\\\"", '; - put ' [ "\b" ] = "\\b", '; - put ' [ "\f" ] = "\\f", '; - put ' [ "\n" ] = "\\n", '; - put ' [ "\r" ] = "\\r", '; - put ' [ "\t" ] = "\\t", '; - put '} '; - put ' '; - put 'local escape_char_map_inv = { [ "\\/" ] = "/" } '; - put 'for k, v in pairs(escape_char_map) do '; - put ' escape_char_map_inv[v] = k '; - put 'end '; - put ' '; - put 'local function escape_char(c) '; - put ' return escape_char_map[c] or string.format("\\u%04x", c:byte()) '; - put 'end '; - put ' '; - put 'local function encode_nil(val) '; - put ' return "null" '; - put 'end '; - put ' '; - put 'local function encode_table(val, stack) '; - put ' local res = {} '; - put ' stack = stack or {} '; - put ' '; - put ' -- Circular reference? '; - put ' if stack[val] then error("circular reference") end '; - put ' '; - put ' stack[val] = true '; - put ' '; - put ' if rawget(val, 1) ~= nil or next(val) == nil then '; - put ' -- Treat as array -- check keys are valid and it is not sparse '; - put ' local n = 0 '; - put ' for k in pairs(val) do '; - put ' if type(k) ~= "number" then '; - put ' error("invalid table: mixed or invalid key types") '; - put ' end '; - put ' n = n + 1 '; - put ' end '; - put ' if n ~= #val then '; - put ' error("invalid table: sparse array") '; - put ' end '; - put ' -- Encode '; - put ' for i, v in ipairs(val) do '; - put ' table.insert(res, encode(v, stack)) '; - put ' end '; - put ' stack[val] = nil '; - put ' return "[" .. table.concat(res, ",") .. "]" '; - put ' else '; - put ' -- Treat as an object '; - put ' for k, v in pairs(val) do '; - put ' if type(k) ~= "string" then '; - put ' error("invalid table: mixed or invalid key types") '; - put ' end '; - put ' table.insert(res, encode(k, stack) .. ":" .. encode(v, stack)) '; - put ' end '; - put ' stack[val] = nil '; - put ' return "{" .. table.concat(res, ",") .. "}" '; - put ' end '; - put 'end '; - put ' '; - put 'local function encode_string(val) '; - put ' return ''"'' .. val:gsub(''[%z\1-\31\\"]'', escape_char) .. ''"'' '; - put 'end '; - put ' '; - put 'local function encode_number(val) '; - put ' -- Check for NaN, -inf and inf '; - put ' if val ~= val or val <= -math.huge or val >= math.huge then '; - put ' error("unexpected number value ''" .. tostring(val) .. "''") '; - put ' end '; - put ' return string.format("%.14g", val) '; - put 'end '; - put ' '; - put 'local type_func_map = { '; - put ' [ "nil" ] = encode_nil, '; - put ' [ "table" ] = encode_table, '; - put ' [ "string" ] = encode_string, '; - put ' [ "number" ] = encode_number, '; - put ' [ "boolean" ] = tostring, '; - put '} '; - put ' '; - put 'encode = function(val, stack) '; - put ' local t = type(val) '; - put ' local f = type_func_map[t] '; - put ' if f then '; - put ' return f(val, stack) '; - put ' end '; - put ' error("unexpected type ''" .. t .. "''") '; - put 'end '; - put ' '; - put 'function json.encode(val) '; - put ' return ( encode(val) ) '; - put 'end '; - put ' '; - put '------------------------------------------------------------------------------- '; - put '-- Decode '; - put '------------------------------------------------------------------------------- '; - put 'local parse '; - put 'local function create_set(...) '; - put ' local res = {} '; - put ' for i = 1, select("#", ...) do '; - put ' res[ select(i, ...) ] = true '; - put ' end '; - put ' return res '; - put 'end '; - put ' '; - put 'local space_chars = create_set(" ", "\t", "\r", "\n") '; - put 'local delim_chars = create_set(" ", "\t", "\r", "\n", "]", "}", ",") '; - put 'local escape_chars = create_set("\\", "/", ''"'', "b", "f", "n", "r", "t", "u") '; - put 'local literals = create_set("true", "false", "null") '; - put ' '; - put 'local literal_map = { '; - put ' [ "true" ] = true, '; - put ' [ "false" ] = false, '; - put ' [ "null" ] = nil, '; - put '} '; - put ' '; - put 'local function next_char(str, idx, set, negate) '; - put ' for i = idx, #str do '; - put ' if set[str:sub(i, i)] ~= negate then '; - put ' return i '; - put ' end '; - put ' end '; - put ' return #str + 1 '; - put 'end '; - put ' '; - put 'local function decode_error(str, idx, msg) '; - put ' local line_count = 1 '; - put ' local col_count = 1 '; - put ' for i = 1, idx - 1 do '; - put ' col_count = col_count + 1 '; - put ' if str:sub(i, i) == "\n" then '; - put ' line_count = line_count + 1 '; - put ' col_count = 1 '; - put ' end '; - put ' end '; - put ' error( string.format("%s at line %d col %d", msg, line_count, col_count) ) '; - put 'end '; - put ' '; - put 'local function codepoint_to_utf8(n) '; - put ' -- http://scripts.sil.org/cms/scripts/page.php?site_id=nrsi&id=iws-appendixa '; - put ' local f = math.floor '; - put ' if n <= 0x7f then '; - put ' return string.char(n) '; - put ' elseif n <= 0x7ff then '; - put ' return string.char(f(n / 64) + 192, n % 64 + 128) '; - put ' elseif n <= 0xffff then '; - put ' return string.char(f(n / 4096) + 224, f(n % 4096 / 64) + 128, n % 64 + 128) '; - put ' elseif n <= 0x10ffff then '; - put ' return string.char(f(n / 262144) + 240, f(n % 262144 / 4096) + 128, '; - put ' f(n % 4096 / 64) + 128, n % 64 + 128) '; - put ' end '; - put ' error( string.format("invalid unicode codepoint ''%x''", n) ) '; - put 'end '; - put ' '; - put 'local function parse_unicode_escape(s) '; - put ' local n1 = tonumber( s:sub(3, 6), 16 ) '; - put ' local n2 = tonumber( s:sub(9, 12), 16 ) '; - put ' -- Surrogate pair? '; - put ' if n2 then '; - put ' return codepoint_to_utf8((n1 - 0xd800) * 0x400 + (n2 - 0xdc00) + 0x10000) '; - put ' else '; - put ' return codepoint_to_utf8(n1) '; - put ' end '; - put 'end '; - put ' '; - put 'local function parse_string(str, i) '; - put ' local has_unicode_escape = false '; - put ' local has_surrogate_escape = false '; - put ' local has_escape = false '; - put ' local last '; - put ' for j = i + 1, #str do '; - put ' local x = str:byte(j) '; - put ' if x < 32 then '; - put ' decode_error(str, j, "control character in string") '; - put ' end '; - put ' if last == 92 then -- "\\" (escape char) '; - put ' if x == 117 then -- "u" (unicode escape sequence) '; - put ' local hex = str:sub(j + 1, j + 5) '; - put ' if not hex:find("%x%x%x%x") then '; - put ' decode_error(str, j, "invalid unicode escape in string") '; - put ' end '; - put ' if hex:find("^[dD][89aAbB]") then '; - put ' has_surrogate_escape = true '; - put ' else '; - put ' has_unicode_escape = true '; - put ' end '; - put ' else '; - put ' local c = string.char(x) '; - put ' if not escape_chars[c] then '; - put ' decode_error(str, j, "invalid escape char ''" .. c .. "'' in string") '; - put ' end '; - put ' has_escape = true '; - put ' end '; - put ' last = nil '; - put ' elseif x == 34 then -- ''"'' (end of string) '; - put ' local s = str:sub(i + 1, j - 1) '; - put ' if has_surrogate_escape then '; - put ' s = s:gsub("\\u[dD][89aAbB]..\\u....", parse_unicode_escape) '; - put ' end '; - put ' if has_unicode_escape then '; - put ' s = s:gsub("\\u....", parse_unicode_escape) '; - put ' end '; - put ' if has_escape then '; - put ' s = s:gsub("\\.", escape_char_map_inv) '; - put ' end '; - put ' return s, j + 1 '; - put ' else '; - put ' last = x '; - put ' end '; - put ' end '; - put ' decode_error(str, i, "expected closing quote for string") '; - put 'end '; - put ' '; - put 'local function parse_number(str, i) '; - put ' local x = next_char(str, i, delim_chars) '; - put ' local s = str:sub(i, x - 1) '; - put ' local n = tonumber(s) '; - put ' if not n then '; - put ' decode_error(str, i, "invalid number ''" .. s .. "''") '; - put ' end '; - put ' return n, x '; - put 'end '; - put ' '; - put 'local function parse_literal(str, i) '; - put ' local x = next_char(str, i, delim_chars) '; - put ' local word = str:sub(i, x - 1) '; - put ' if not literals[word] then '; - put ' decode_error(str, i, "invalid literal ''" .. word .. "''") '; - put ' end '; - put ' return literal_map[word], x '; - put 'end '; - put ' '; - put 'local function parse_array(str, i) '; - put ' local res = {} '; - put ' local n = 1 '; - put ' i = i + 1 '; - put ' while 1 do '; - put ' local x '; - put ' i = next_char(str, i, space_chars, true) '; - put ' -- Empty / end of array? '; - put ' if str:sub(i, i) == "]" then '; - put ' i = i + 1 '; - put ' break '; - put ' end '; - put ' -- Read token '; - put ' x, i = parse(str, i) '; - put ' res[n] = x '; - put ' n = n + 1 '; - put ' -- Next token '; - put ' i = next_char(str, i, space_chars, true) '; - put ' local chr = str:sub(i, i) '; - put ' i = i + 1 '; - put ' if chr == "]" then break end '; - put ' if chr ~= "," then decode_error(str, i, "expected '']'' or '',''") end '; - put ' end '; - put ' return res, i '; - put 'end '; - put ' '; - put 'local function parse_object(str, i) '; - put ' local res = {} '; - put ' i = i + 1 '; - put ' while 1 do '; - put ' local key, val '; - put ' i = next_char(str, i, space_chars, true) '; - put ' -- Empty / end of object? '; - put ' if str:sub(i, i) == "}" then '; - put ' i = i + 1 '; - put ' break '; - put ' end '; - put ' -- Read key '; - put ' if str:sub(i, i) ~= ''"'' then '; - put ' decode_error(str, i, "expected string for key") '; - put ' end '; - put ' key, i = parse(str, i) '; - put ' -- Read '':'' delimiter '; - put ' i = next_char(str, i, space_chars, true) '; - put ' if str:sub(i, i) ~= ":" then '; - put ' decode_error(str, i, "expected '':'' after key") '; - put ' end '; - put ' i = next_char(str, i + 1, space_chars, true) '; - put ' -- Read value '; - put ' val, i = parse(str, i) '; - put ' -- Set '; - put ' res[key] = val '; - put ' -- Next token '; - put ' i = next_char(str, i, space_chars, true) '; - put ' local chr = str:sub(i, i) '; - put ' i = i + 1 '; - put ' if chr == "}" then break end '; - put ' if chr ~= "," then decode_error(str, i, "expected ''}'' or '',''") end '; - put ' end '; - put ' return res, i '; - put 'end '; - put ' '; - put 'local char_func_map = { '; - put ' [ ''"'' ] = parse_string, '; - put ' [ "0" ] = parse_number, '; - put ' [ "1" ] = parse_number, '; - put ' [ "2" ] = parse_number, '; - put ' [ "3" ] = parse_number, '; - put ' [ "4" ] = parse_number, '; - put ' [ "5" ] = parse_number, '; - put ' [ "6" ] = parse_number, '; - put ' [ "7" ] = parse_number, '; - put ' [ "8" ] = parse_number, '; - put ' [ "9" ] = parse_number, '; - put ' [ "-" ] = parse_number, '; - put ' [ "t" ] = parse_literal, '; - put ' [ "f" ] = parse_literal, '; - put ' [ "n" ] = parse_literal, '; - put ' [ "[" ] = parse_array, '; - put ' [ "{" ] = parse_object, '; - put '} '; - put ' '; - put 'parse = function(str, idx) '; - put ' local chr = str:sub(idx, idx) '; - put ' local f = char_func_map[chr] '; - put ' if f then '; - put ' return f(str, idx) '; - put ' end '; - put ' decode_error(str, idx, "unexpected character ''" .. chr .. "''") '; - put 'end '; - put ' '; - put 'function json.decode(str) '; - put ' if type(str) ~= "string" then '; - put ' error("expected argument of type string, got " .. type(str)) '; - put ' end '; - put ' local res, idx = parse(str, next_char(str, 1, space_chars, true)) '; - put ' idx = next_char(str, idx, space_chars, true) '; - put ' if idx <= #str then '; - put ' decode_error(str, idx, "trailing garbage") '; - put ' end '; - put ' return res '; - put 'end '; - put ' '; - put 'return json '; - put ' '; - put '-- JSON.LUA ENDS HERE '; -run; - -/* ensure big enough lrecl to avoid lua compilation issues */ -%local optval; -%let optval=%sysfunc(getoption(lrecl)); -options lrecl=1024; - -/* execute the lua code by using a .lua extension */ -%inc "%sysfunc(pathname(work))/ml_json.lua" /source2; - -options lrecl=&optval; - -%mend ml_json; diff --git a/008_macros/mmx_createmetafolder.sas b/008_macros/mmx_createmetafolder.sas deleted file mode 100644 index 38beb35..0000000 --- a/008_macros/mmx_createmetafolder.sas +++ /dev/null @@ -1,50 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a metadata folder - @details Creates a metadata folder using the batch tools - - Usage: - - %mmx_createmetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345) - -

SAS Macros

- @li mf_loc.sas - @li mp_abort.sas - - @param [in] loc= the metadata folder to delete - @param [in] user= username - @param [in] pass= password - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mmx_createmetafolder(loc=,user=,pass=); - -%local host port path connx_string msg; -%let host=%sysfunc(getoption(metaserver)); -%let port=%sysfunc(getoption(metaport)); -%let path=%mf_loc(POF)/tools; - -%let connx_string= -host &host -port &port -user '&user' -password '&pass'; -/* remove directory */ -data _null_; - infile " &path/sas-make-folder &connx_string ""&loc"" -makeFullPath 2>&1" - pipe lrecl=10000; - input; - putlog _infile_; -run; - -data _null_; /* check tree exists */ - length type uri $256; - rc=metadata_pathobj("","&loc","Folder",type,uri); - call symputx('foldertype',type,'l'); -run; -%let msg=Location (&loc) was not created!!; -%mp_abort(iftrue= (&foldertype ne Tree) - ,mac=&_program..sas - ,msg=%superq(msg) -) - -%mend mmx_createmetafolder; diff --git a/008_macros/mmx_deletemetafolder.sas b/008_macros/mmx_deletemetafolder.sas deleted file mode 100644 index 762bbe3..0000000 --- a/008_macros/mmx_deletemetafolder.sas +++ /dev/null @@ -1,40 +0,0 @@ -/*** HELP START ***//** - @file - @brief Deletes a metadata folder - @details Deletes a metadata folder (and contents) using the batch tools, as - documented here: - https://documentation.sas.com/?docsetId=bisag&docsetTarget=p0zqp8fmgs4o0kn1tt7j8ho829fv.htm&docsetVersion=9.4&locale=en - - Usage: - - %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345) - -

SAS Macros

- @li mf_loc.sas - - @param [in] loc= the metadata folder to delete - @param [in] user= username - @param [in] pass= password - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mmx_deletemetafolder(loc=,user=,pass=); - -%local host port path connx_string; -%let host=%sysfunc(getoption(metaserver)); -%let port=%sysfunc(getoption(metaport)); -%let path=%mf_loc(POF)/tools; - -%let connx_string= -host &host -port &port -user '&user' -password '&pass'; -/* remove directory */ -data _null_; - infile " &path/sas-delete-objects &connx_string ""&loc"" -deleteContents 2>&1" - pipe lrecl=10000; - input; - putlog _infile_; -run; - -%mend mmx_deletemetafolder; diff --git a/008_macros/mmx_spkexport.sas b/008_macros/mmx_spkexport.sas deleted file mode 100644 index ddde21a..0000000 --- a/008_macros/mmx_spkexport.sas +++ /dev/null @@ -1,94 +0,0 @@ -/*** HELP START ***//** - @file mmx_spkexport.sas - @brief Exports everything in a particular metadata folder - @details Will export everything in a metadata folder to a specified location. - Note - the batch tools require a username and password. For security, - these are expected to have been provided in a protected directory. - -Usage: - - %* import the macros (or make them available some other way); - filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %* create sample text file as input to the macro; - filename tmp temp; - data _null_; - file tmp; - put '%let mmxuser=sasdemo;'; - put '%let mmxpass=Mars321'; - run; - - filename outref "%sysfunc(pathname(work))"; - %mmx_spkexport( - metaloc=%str(/30.Projects/3001.Internal/300115.DataController/dc1) - ,secureref=tmp - ,outspkpath=%str(/tmp) - ) - -

SAS Macros

- @li mf_loc.sas - @li mm_tree.sas - @li mf_getuniquefileref.sas - @li mp_abort.sas - - @param [in] metaloc= the metadata folder to export - @param [in] secureref= () fileref containing the username / password - (should point to a file in a secure location) - @param [in] outspkname= name of the spk to be created (default is mmxport). - @param [in] outspkpath= ((%sysfunc(pathname(WORK))) - directory in which to create the SPK. - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mmx_spkexport(metaloc= - ,secureref= - ,outspkname=mmxport - ,outspkpath=%sysfunc(pathname(work)) -); - -%local host port platform_object_path connx_string; -%let host=%sysfunc(getoption(metaserver)); -%let port=%sysfunc(getoption(metaport)); -%let platform_object_path=%mf_loc(POF); - -/* get creds */ -%inc &secureref/nosource; - -%let connx_string= - %str(-host &host -port &port -user '&mmxuser' -password '&mmxpass'); - -%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable) - -%local fref1; -%let fref1=%mf_getuniquefileref(); -data ; - set exportable end=last; - file &fref1 lrecl=32767; - length str $32767; - if _n_=1 then do; - put 'data _null_;'; - put 'infile "cd ""&platform_object_path"" %trim('; - put ') cd ""&platform_object_path"" %trim('; - put '); ./ExportPackage &connx_string -disableX11 %trim('; - put ') -package ""&outspkpath/&outspkname..spk"" %trim('; - end; - str=') -objects '!!cats('""',path,'/',name,"(",publictype,')"" %trim('); - put str; - if last then do; - put ') -log ""&outspkpath/&outspkname..log"" 2>&1" pipe lrecl=10000;'; - put 'input;putlog _infile_;run;'; - end; -run; - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc) -) - -%inc &fref1; - -%mend mmx_spkexport; diff --git a/001_macros/mv_createfile.sas b/008_macros/mv_createfile.sas similarity index 100% rename from 001_macros/mv_createfile.sas rename to 008_macros/mv_createfile.sas diff --git a/001_macros/mv_createfolder.sas b/008_macros/mv_createfolder.sas similarity index 100% rename from 001_macros/mv_createfolder.sas rename to 008_macros/mv_createfolder.sas diff --git a/001_macros/mv_createjob.sas b/008_macros/mv_createjob.sas similarity index 100% rename from 001_macros/mv_createjob.sas rename to 008_macros/mv_createjob.sas diff --git a/001_macros/mv_createwebservice.sas b/008_macros/mv_createwebservice.sas similarity index 100% rename from 001_macros/mv_createwebservice.sas rename to 008_macros/mv_createwebservice.sas diff --git a/001_macros/mv_deletefoldermember.sas b/008_macros/mv_deletefoldermember.sas similarity index 100% rename from 001_macros/mv_deletefoldermember.sas rename to 008_macros/mv_deletefoldermember.sas diff --git a/001_macros/mv_deletejes.sas b/008_macros/mv_deletejes.sas similarity index 100% rename from 001_macros/mv_deletejes.sas rename to 008_macros/mv_deletejes.sas diff --git a/001_macros/mv_deleteviyafolder.sas b/008_macros/mv_deleteviyafolder.sas similarity index 100% rename from 001_macros/mv_deleteviyafolder.sas rename to 008_macros/mv_deleteviyafolder.sas diff --git a/001_macros/mv_getclients.sas b/008_macros/mv_getclients.sas similarity index 100% rename from 001_macros/mv_getclients.sas rename to 008_macros/mv_getclients.sas diff --git a/001_macros/mv_getfoldermembers.sas b/008_macros/mv_getfoldermembers.sas similarity index 100% rename from 001_macros/mv_getfoldermembers.sas rename to 008_macros/mv_getfoldermembers.sas diff --git a/001_macros/mv_getgroupmembers.sas b/008_macros/mv_getgroupmembers.sas similarity index 100% rename from 001_macros/mv_getgroupmembers.sas rename to 008_macros/mv_getgroupmembers.sas diff --git a/001_macros/mv_getgroups.sas b/008_macros/mv_getgroups.sas similarity index 100% rename from 001_macros/mv_getgroups.sas rename to 008_macros/mv_getgroups.sas diff --git a/001_macros/mv_getjobcode.sas b/008_macros/mv_getjobcode.sas similarity index 100% rename from 001_macros/mv_getjobcode.sas rename to 008_macros/mv_getjobcode.sas diff --git a/001_macros/mv_getjoblog.sas b/008_macros/mv_getjoblog.sas similarity index 100% rename from 001_macros/mv_getjoblog.sas rename to 008_macros/mv_getjoblog.sas diff --git a/001_macros/mv_getjobresult.sas b/008_macros/mv_getjobresult.sas similarity index 100% rename from 001_macros/mv_getjobresult.sas rename to 008_macros/mv_getjobresult.sas diff --git a/001_macros/mv_getjobstate.sas b/008_macros/mv_getjobstate.sas similarity index 100% rename from 001_macros/mv_getjobstate.sas rename to 008_macros/mv_getjobstate.sas diff --git a/001_macros/mv_getusergroups.sas b/008_macros/mv_getusergroups.sas similarity index 100% rename from 001_macros/mv_getusergroups.sas rename to 008_macros/mv_getusergroups.sas diff --git a/001_macros/mv_getusers.sas b/008_macros/mv_getusers.sas similarity index 100% rename from 001_macros/mv_getusers.sas rename to 008_macros/mv_getusers.sas diff --git a/001_macros/mv_jobexecute.sas b/008_macros/mv_jobexecute.sas similarity index 100% rename from 001_macros/mv_jobexecute.sas rename to 008_macros/mv_jobexecute.sas diff --git a/001_macros/mv_jobflow.sas b/008_macros/mv_jobflow.sas similarity index 100% rename from 001_macros/mv_jobflow.sas rename to 008_macros/mv_jobflow.sas diff --git a/001_macros/mv_jobwaitfor.sas b/008_macros/mv_jobwaitfor.sas similarity index 100% rename from 001_macros/mv_jobwaitfor.sas rename to 008_macros/mv_jobwaitfor.sas diff --git a/001_macros/mv_registerclient.sas b/008_macros/mv_registerclient.sas similarity index 100% rename from 001_macros/mv_registerclient.sas rename to 008_macros/mv_registerclient.sas diff --git a/001_macros/mv_tokenauth.sas b/008_macros/mv_tokenauth.sas similarity index 100% rename from 001_macros/mv_tokenauth.sas rename to 008_macros/mv_tokenauth.sas diff --git a/001_macros/mv_tokenrefresh.sas b/008_macros/mv_tokenrefresh.sas similarity index 100% rename from 001_macros/mv_tokenrefresh.sas rename to 008_macros/mv_tokenrefresh.sas diff --git a/001_macros/mv_webout.sas b/008_macros/mv_webout.sas similarity index 100% rename from 001_macros/mv_webout.sas rename to 008_macros/mv_webout.sas diff --git a/009_macros/mf_abort.sas b/009_macros/mf_abort.sas deleted file mode 100644 index 6b64b4c..0000000 --- a/009_macros/mf_abort.sas +++ /dev/null @@ -1,32 +0,0 @@ -/*** HELP START ***//** - @file - @brief Abort, ungracefully - @details Will abort with a straightforward %abort if the condition is true. - - @param [in] mac= (mf_abort.sas) Name of calling macro (is printed to the log) - @param [in] msg= ( ) Additional string to print to the log - @param [in] iftrue= (%str(1=1)) Conditional logic under which to perform the - abort - -

Related Macros

- @li mp_abort.sas - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mf_abort(mac=mf_abort.sas, msg=, iftrue=%str(1=1) -)/des='ungraceful abort' /*STORE SOURCE*/; - - %if not(%eval(%unquote(&iftrue))) %then %return; - - %put NOTE: /// mf_abort macro executing //; - %if %length(&mac)>0 %then %put NOTE- called by &mac; - %put NOTE - &msg; - - %abort; - -%mend mf_abort; - -/** @endcond */ diff --git a/009_macros/mf_dedup.sas b/009_macros/mf_dedup.sas deleted file mode 100644 index b335f49..0000000 --- a/009_macros/mf_dedup.sas +++ /dev/null @@ -1,53 +0,0 @@ -/*** HELP START ***//** - @file - @brief de-duplicates a macro string - @details Removes all duplicates from a string of words. A delimeter can be - chosen. Is inspired heavily by this excellent [macro]( - https://github.com/scottbass/SAS/blob/master/Macro/dedup_mstring.sas) from - [Scott Base](https://www.linkedin.com/in/scottbass). Case sensitive. - - Usage: - - %let str=One two one two and through and through; - %put %mf_dedup(&str); - %put %mf_dedup(&str,outdlm=%str(,)); - - Which returns: - - > One two one and through - > One,two,one,and,through - - @param [in] str String to be deduplicated - @param [in] indlm= ( ) Delimeter of the input string - @param [out] outdlm= ( ) Delimiter of the output string - -

Related Macros

- @li mf_trimstr.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_dedup(str - ,indlm=%str( ) - ,outdlm=%str( ) -)/*/STORE SOURCE*/; - -%local num word i pos out; - -%* loop over each token, searching the target for that token ; -%let num=%sysfunc(countc(%superq(str),%str(&indlm))); -%do i=1 %to %eval(&num+1); - %let word=%scan(%superq(str),&i,%str(&indlm)); - %let pos=%sysfunc(indexw(&out,&word,%str(&outdlm))); - %if (&pos eq 0) %then %do; - %if (&i gt 1) %then %let out=&out%str(&outdlm); - %let out=&out&word; - %end; -%end; - -%unquote(&out) - -%mend mf_dedup; - - diff --git a/009_macros/mf_deletefile.sas b/009_macros/mf_deletefile.sas deleted file mode 100644 index e271f67..0000000 --- a/009_macros/mf_deletefile.sas +++ /dev/null @@ -1,31 +0,0 @@ -/*** HELP START ***//** - @file - @brief Deletes a physical file, if it exists - @details Usage: - - %mf_writefile(&sasjswork/myfile.txt,l1=some content) - - %mf_deletefile(&sasjswork/myfile.txt) - - %mf_deletefile(&sasjswork/myfile.txt) - - - @param [in] file Full path to the target file - - @returns The return code from the fdelete() invocation - -

Related Macros

- @li mf_deletefile.test.sas - @li mf_writefile.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_deletefile(file -)/*/STORE SOURCE*/; - %local rc fref; - %let rc= %sysfunc(filename(fref,&file)); - %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg()); - %let rc= %sysfunc(filename(fref)); -%mend mf_deletefile; diff --git a/009_macros/mf_existds.sas b/009_macros/mf_existds.sas deleted file mode 100644 index 1c06ff2..0000000 --- a/009_macros/mf_existds.sas +++ /dev/null @@ -1,30 +0,0 @@ -/*** HELP START ***//** - @file mf_existds.sas - @brief Checks whether a dataset OR a view exists. - @details Can be used in open code, eg as follows: - - %if %mf_existds(libds=work.someview) %then %put yes it does!; - - NOTE - some databases have case sensitive tables, for instance POSTGRES - with the preserve_tab_names=yes libname setting. This may impact - expected results (depending on whether you 'expect' the result to be - case insensitive in this context!) - - @param [in] libds library.dataset - @return output returns 1 or 0 - -

Related Macros

- @li mf_existds.test.sas - - @warning Untested on tables registered in metadata but not physically present - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_existds(libds -)/*/STORE SOURCE*/; - - %if %sysfunc(exist(&libds)) ne 1 & %sysfunc(exist(&libds,VIEW)) ne 1 %then 0; - %else 1; - -%mend mf_existds; diff --git a/009_macros/mf_existfeature.sas b/009_macros/mf_existfeature.sas deleted file mode 100644 index e447425..0000000 --- a/009_macros/mf_existfeature.sas +++ /dev/null @@ -1,59 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a feature exists - @details Check to see if a feature is supported in your environment. - Run without arguments to see a list of detectable features. - Note - this list is based on known versions of SAS rather than - actual feature detection, as that is tricky / impossible to do - without generating errs in most cases. - - %put %mf_existfeature(PROCLUA); - - @param [in] feature The feature to detect. - - @return output returns 1 or 0 (or -1 if not found) - -

SAS Macros

- @li mf_getplatform.sas - - @version 8 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ -%macro mf_existfeature(feature -)/*/STORE SOURCE*/; - %let feature=%upcase(&feature); - %local platform; - %let platform=%mf_getplatform(); - - %if &feature= %then %do; - %put No feature was requested for detection; - %end; - %else %if &feature=COLCONSTRAINTS %then %do; - %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0; - %else 1; - %end; - %else %if &feature=PROCLUA %then %do; - /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */ - %if &platform=SASVIYA %then 1; - %else %if "&sysver"="9.2" or "&sysver"="9.3" %then 0; - %else %if "&SYSVLONG" < "9.04.01M3" %then 0; - %else 1; - %end; - %else %if &feature=DBMS_MEMTYPE %then %do; - /* does dbms_memtype exist in dictionary.tables? */ - %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0; - %else 1; - %end; - %else %if &feature=EXPORTXLS %then %do; - /* is it possible to PROC EXPORT an excel file? */ - %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1; - %else %if %sysfunc(sysprod(SAS/ACCESS Interface to PC Files)) = 1 %then 1; - %else 0; - %end; - %else %do; - -1 - %put &sysmacroname: &feature not found; - %end; -%mend mf_existfeature; -/** @endcond */ diff --git a/009_macros/mf_existfileref.sas b/009_macros/mf_existfileref.sas deleted file mode 100644 index f78a774..0000000 --- a/009_macros/mf_existfileref.sas +++ /dev/null @@ -1,33 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a fileref exists - @details You can probably do without this macro as it is just a one liner. - Mainly it is here as a convenient way to remember the syntax! - - @param [in] fref the fileref to detect - - @return output Returns 1 if found and 0 if not found. Note - it is possible - that the fileref is found, but the file does not (yet) exist. If you need - to test for this, you may as well use the fileref function directly. - - @version 8 - @author [Allan Bowe](https://www.linkedin.com/in/allanbowe/) -**//*** HELP END ***/ - -%macro mf_existfileref(fref -)/*/STORE SOURCE*/; - - %local rc; - %let rc=%sysfunc(fileref(&fref)); - %if &rc=0 %then %do; - 1 - %end; - %else %if &rc<0 %then %do; - %put &sysmacroname: Fileref &fref exists but the underlying file does not; - 1 - %end; - %else %do; - 0 - %end; - -%mend mf_existfileref; diff --git a/009_macros/mf_existfunction.sas b/009_macros/mf_existfunction.sas deleted file mode 100644 index cae475b..0000000 --- a/009_macros/mf_existfunction.sas +++ /dev/null @@ -1,37 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks if a function exists - @details Returns 1 if the function exists, else 0. Note that this function - can be slow as it needs to open the sashelp.vfuncs table. - - Usage: - - %put %mf_existfunction(CAT); - %put %mf_existfunction(DOG); - - Full credit to [Bart](https://sasensei.com/user/305) for the vfunc pointer - and the tidy approach for pure macro data set filtering. - Check out his [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) - framework! Where you can find the same [function]( -https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionexists-macro - ). - - @param [in] name function name - - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ -%macro mf_existfunction(name -)/*/STORE SOURCE*/; - - %local dsid rc exist; - %let dsid=%sysfunc(open(sashelp.vfunc(where=(fncname="%upcase(&name)")))); - %let exist=1; - %let exist=%sysfunc(fetch(&dsid, NOSET)); - %let rc=%sysfunc(close(&dsid)); - - %sysevalf(0 = &exist) - -%mend mf_existfunction; - -/** @endcond */ diff --git a/009_macros/mf_existvar.sas b/009_macros/mf_existvar.sas deleted file mode 100644 index 8498421..0000000 --- a/009_macros/mf_existvar.sas +++ /dev/null @@ -1,43 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks if a variable exists in a data set. - @details Returns 0 if the variable does NOT exist, and the position of the var - if it does. - Usage: - - %put %mf_existvar(work.someds, somevar) - - @param [in] libds 2 part dataset or view reference - @param [in] var variable name - -

Related Macros

- @li mf_existvar.test.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ - -%macro mf_existvar(libds /* 2 part dataset name */ - , var /* variable name */ -)/*/STORE SOURCE*/; - - %local dsid rc; - %let dsid=%sysfunc(open(&libds,is)); - - %if &dsid=0 %then %do; - %put %sysfunc(sysmsg()); - 0 - %end; - %else %if %length(&var)=0 %then %do; - 0 - %let rc=%sysfunc(close(&dsid)); - %end; - %else %do; - %sysfunc(varnum(&dsid,&var)) - %let rc=%sysfunc(close(&dsid)); - %end; - -%mend mf_existvar; - -/** @endcond */ diff --git a/009_macros/mf_existvarlist.sas b/009_macros/mf_existvarlist.sas deleted file mode 100644 index 3a2d2b3..0000000 --- a/009_macros/mf_existvarlist.sas +++ /dev/null @@ -1,56 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks if a set of variables ALL exist in a data set. - @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do. - Usage: - - %put %mf_existVarList(sashelp.class, age sex name dummyvar); - - @param [in] libds 2 part dataset or view reference - @param [in] varlist space separated variable names - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mf_existvarlist(libds, varlist -)/*/STORE SOURCE*/; - - %if %str(&libds)=%str() or %str(&varlist)=%str() %then %do; - %mf_abort(msg=No value provided to libds(&libds) or varlist (&varlist)! - ,mac=mf_existvarlist.sas) - %end; - - %local dsid rc i var found; - %let dsid=%sysfunc(open(&libds,is)); - - %if &dsid=0 %then %do; - %put %str(WARN)ING: unable to open &libds in mf_existvarlist (&dsid); - %end; - - %if %sysfunc(attrn(&dsid,NVARS))=0 %then %do; - %put MF_EXISTVARLIST: No variables in &libds ; - 0 - %return; - %end; - - %else %do i=1 %to %sysfunc(countw(&varlist)); - %let var=%scan(&varlist,&i); - - %if %sysfunc(varnum(&dsid,&var))=0 %then %do; - %let found=&found &var; - %end; - %end; - - %let rc=%sysfunc(close(&dsid)); - %if %str(&found)=%str() %then %do; - 1 - %end; - %else %do; - 0 - %put Vars not found: &found; - %end; -%mend mf_existvarlist; - -/** @endcond */ diff --git a/009_macros/mf_fmtdttm.sas b/009_macros/mf_fmtdttm.sas deleted file mode 100644 index 7602c1d..0000000 --- a/009_macros/mf_fmtdttm.sas +++ /dev/null @@ -1,42 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns E8601DT26.6 if compatible else DATETIME19.3 - @details From our experience in [Data Controller for SAS] - (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has - the widest support when it comes to pass-through SQL queries. - - However, it is not supported in WPS or early versions of SAS 9 (M3 and below) - when used as a datetime literal, eg: - - data _null_; - demo="%sysfunc(datetime(),E8601DT26.6)"dt; - demo=; - run; - - This macro will therefore return DATEITME19.3 as an alternative format - based on the runtime environment so that it can be used in such cases, eg: - - data _null_; - demo="%sysfunc(datetime(),%mf_fmtdttm())"dt; - demo=; - run; - -

Related Macros

- @li mf_fmtdttm.test.sas - - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_fmtdttm( -)/*/STORE SOURCE*/; - -%if "&sysver"="9.2" or "&sysver"="9.3" - or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3") - or "%substr(&sysver,1,1)"="4" - or "%substr(&sysver,1,1)"="5" -%then %do;DATETIME19.3%end; -%else %do;E8601DT26.6%end; - -%mend mf_fmtdttm; - - diff --git a/009_macros/mf_getapploc.sas b/009_macros/mf_getapploc.sas deleted file mode 100644 index 76702b1..0000000 --- a/009_macros/mf_getapploc.sas +++ /dev/null @@ -1,79 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the appLoc from the _program variable - @details When working with SASjs apps, web services / tests / jobs are always - deployed to a root (app) location in the SAS logical folder tree. - - When building apps for use in other environments, you do not necessarily know - where the backend services will be deployed. Therefore a function like this - is handy in order to dynamically figure out the appLoc, and enable other - services to be connected by a relative reference. - - SASjs apps always have the same immediate substructure (one or more of the - following): - - @li /data - @li /jobs - @li /services - @li /tests - @li /tests/jobs - @li /tests/services - @li /tests/macros - - This function works by testing for the existence of any of the above in the - automatic _program variable, and returning the part to the left of it. - - Usage: - - %put %mf_getapploc(&_program) - - %put %mf_getapploc(/some/location/services/admin/myservice); - %put %mf_getapploc(/some/location/jobs/extract/somejob/); - %put %mf_getapploc(/some/location/tests/jobs/somejob/); - - @param [in] pgm The _program value from which to extract the appLoc - - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getapploc(pgm); -%if "&pgm"="" %then %do; - %if %symexist(_program) %then %let pgm=&_program; - %else %do; - %put &sysmacroname: No value provided and no _program variable available; - %return; - %end; -%end; -%local root; - -/** - * First check we are not in the tests/macros folder (which has no subfolders) - * or specifically in the testsetup or testteardown services - */ -%if %index(&pgm,/tests/macros/) -or %index(&pgm,/tests/testsetup) -or %index(&pgm,/tests/testteardown) -%then %do; - %let root=%substr(&pgm,1,%index(&pgm,/tests)-1); - &root - %return; -%end; - -/** - * Next, move up two levels to avoid matches on subfolder or service name - */ -%let root=%substr(&pgm,1,%length(&pgm)-%length(%scan(&pgm,-1,/))-1); -%let root=%substr(&root,1,%length(&root)-%length(%scan(&root,-1,/))-1); - -%if %index(&root,/tests/) %then %do; - %let root=%substr(&root,1,%index(&root,/tests/)-1); -%end; -%else %if %index(&root,/services) %then %do; - %let root=%substr(&root,1,%index(&root,/services)-1); -%end; -%else %if %index(&root,/jobs) %then %do; - %let root=%substr(&root,1,%index(&root,/jobs)-1); -%end; -%else %put &sysmacroname: Could not find an app location from &pgm; - &root -%mend mf_getapploc ; diff --git a/009_macros/mf_getattrc.sas b/009_macros/mf_getattrc.sas deleted file mode 100644 index af4743a..0000000 --- a/009_macros/mf_getattrc.sas +++ /dev/null @@ -1,34 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a character attribute of a dataset. - @details Can be used in open code, eg as follows: - - %put Dataset label = %mf_getattrc(sashelp.class,LABEL); - %put Member Type = %mf_getattrc(sashelp.class,MTYPE); - - @param [in] libds library.dataset - @param [in] attr full list in [documentation]( - https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000147794.htm) - @return output returns result of the attrc value supplied, or -1 and log - message if err. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getattrc( - libds - ,attr -)/*/STORE SOURCE*/; - %local dsid rc; - %let dsid=%sysfunc(open(&libds,is)); - %if &dsid = 0 %then %do; - %put %str(WARN)ING: Cannot open %trim(&libds), system message below; - %put %sysfunc(sysmsg()); - -1 - %end; - %else %do; - %sysfunc(attrc(&dsid,&attr)) - %let rc=%sysfunc(close(&dsid)); - %end; -%mend mf_getattrc; diff --git a/009_macros/mf_getattrn.sas b/009_macros/mf_getattrn.sas deleted file mode 100644 index 7f8af44..0000000 --- a/009_macros/mf_getattrn.sas +++ /dev/null @@ -1,34 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a numeric attribute of a dataset. - @details Can be used in open code, eg as follows: - - %put Number of observations=%mf_getattrn(sashelp.class,NLOBS); - %put Number of variables = %mf_getattrn(sashelp.class,NVARS); - - @param [in] libds library.dataset - @param [in] attr Common values are NLOBS and NVARS, full list in [documentation]( - http://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000212040.htm) - @return output returns result of the attrn value supplied, or -1 and log - message if err. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getattrn( - libds - ,attr -)/*/STORE SOURCE*/; - %local dsid rc; - %let dsid=%sysfunc(open(&libds,is)); - %if &dsid = 0 %then %do; - %put %str(WARN)ING: Cannot open %trim(&libds), system message below; - %put %sysfunc(sysmsg()); - -1 - %end; - %else %do; - %sysfunc(attrn(&dsid,&attr)) - %let rc=%sysfunc(close(&dsid)); - %end; -%mend mf_getattrn; diff --git a/009_macros/mf_getengine.sas b/009_macros/mf_getengine.sas deleted file mode 100644 index 87741a0..0000000 --- a/009_macros/mf_getengine.sas +++ /dev/null @@ -1,54 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the engine type of a SAS library - @details Usage: - - %put %mf_getengine(SASHELP); - - returns: - > V9 - - A note is also written to the log. The credit for this macro goes to the - contributors of Chris Hemedingers blog [post]( - http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/) - - @param [in] libref Library reference (also accepts a 2 level libds ref). - - @return output returns the library engine (uppercase) for the FIRST library - encountered. - - @warning will only return the FIRST library engine - for concatenated - libraries, with different engines, inconsistent results may be encountered. - - @version 9.2 - @author Allan Bowe - -

Related Macros

- @li mf_getxengine.sas - -**//*** HELP END ***/ -/** @cond */ - -%macro mf_getengine(libref -)/*/STORE SOURCE*/; - %local dsid engnum rc engine; - - /* in case the parameter is a libref.tablename, pull off just the libref */ - %let libref = %upcase(%scan(&libref, 1, %str(.))); - - %let dsid=%sysfunc( - open(sashelp.vlibnam(where=(libname="%upcase(&libref)")),i) - ); - %if (&dsid ^= 0) %then %do; - %let engnum=%sysfunc(varnum(&dsid,ENGINE)); - %let rc=%sysfunc(fetch(&dsid)); - %let engine=%sysfunc(getvarc(&dsid,&engnum)); - %put &libref. ENGINE is &engine.; - %let rc= %sysfunc(close(&dsid)); - %end; - - %upcase(&engine) - -%mend mf_getengine; - -/** @endcond */ diff --git a/009_macros/mf_getfilesize.sas b/009_macros/mf_getfilesize.sas deleted file mode 100644 index fb7b93c..0000000 --- a/009_macros/mf_getfilesize.sas +++ /dev/null @@ -1,64 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the size of a file in bytes. - @details Provide full path/filename.extension to the file, eg: - - %put %mf_getfilesize(fpath=C:\temp\myfile.txt); - - or, provide a libds value as follows: - - data x;do x=1 to 100000;y=x;output;end;run; - %put %mf_getfilesize(libds=work.x,format=yes); - - Which gives: - - > 2mb - - @param [in] fpath= Full path and filename. Provide this OR the libds value. - @param [in] libds= (0) Library.dataset value (assumes library is BASE engine) - @param [in] format= (NO) Set to yes to apply sizekmg. format - - @returns bytes - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getfilesize(fpath=,libds=0,format=NO -)/*/STORE SOURCE*/; - - %local rc fid fref bytes dsid lib vnum; - - %if &libds ne 0 %then %do; - %let libds=%upcase(&libds); - %if %index(&libds,.)=0 %then %let lib=WORK; - %else %let lib=%scan(&libds,1,.); - %let dsid=%sysfunc(open( - sashelp.vtable(where=(libname="&lib" and memname="%scan(&libds,-1,.)") - keep=libname memname filesize - ) - )); - %if (&dsid ^= 0) %then %do; - %let vnum=%sysfunc(varnum(&dsid,FILESIZE)); - %let rc=%sysfunc(fetch(&dsid)); - %let bytes=%sysfunc(getvarn(&dsid,&vnum)); - %let rc= %sysfunc(close(&dsid)); - %end; - %else %put &sysmacroname: &libds could not be opened! %sysfunc(sysmsg()); - %end; - %else %do; - %let rc=%sysfunc(filename(fref,&fpath)); - %let fid=%sysfunc(fopen(&fref)); - %let bytes=%sysfunc(finfo(&fid,File Size (bytes))); - %let rc=%sysfunc(fclose(&fid)); - %let rc=%sysfunc(filename(fref)); - %end; - - %if &format=NO %then %do; - &bytes - %end; - %else %do; - %sysfunc(INPUTN(&bytes, best.),sizekmg.) - %end; - -%mend mf_getfilesize ; diff --git a/009_macros/mf_getfmtlist.sas b/009_macros/mf_getfmtlist.sas deleted file mode 100644 index cd150ae..0000000 --- a/009_macros/mf_getfmtlist.sas +++ /dev/null @@ -1,60 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a distinct list of formats from a table - @details Reads the dataset header and returns a distinct list of formats - applied. - - %put NOTE- %mf_getfmtlist(sashelp.prdsale); - %put NOTE- %mf_getfmtlist(sashelp.shoes); - %put NOTE- %mf_getfmtlist(sashelp.demographics); - - returns: - - DOLLAR $CHAR W MONNAME - $CHAR BEST DOLLAR - BEST Z $CHAR COMMA PERCENTN - - @param [in] libds Two part library.dataset reference. - -

SAS Macros

- @li mf_getfmtname.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_getfmtlist(libds -)/*/STORE SOURCE*/; -/* declare local vars */ -%local out dsid nvars x rc fmt; - -/* open dataset in macro */ -%let dsid=%sysfunc(open(&libds)); - -/* continue if dataset exists */ -%if &dsid %then %do; - /* loop each variable in the dataset */ - %let nvars=%sysfunc(attrn(&dsid,NVARS)); - %do x=1 %to &nvars; - /* grab format and check it exists */ - %let fmt=%sysfunc(varfmt(&dsid,&x)); - %if %quote(&fmt) ne %quote() %then %let fmt=%mf_getfmtname(&fmt); - %else %do; - /* assign default format depending on variable type */ - %if %sysfunc(vartype(&dsid, &x))=C %then %let fmt=$CHAR; - %else %let fmt=BEST; - %end; - /* concatenate unique list of formats */ - %if %sysfunc(indexw(&out,&fmt,%str( )))=0 %then %let out=&out &fmt; - %end; - %let rc=%sysfunc(close(&dsid)); -%end; -%else %do; - %put &sysmacroname: Unable to open &libds (rc=&dsid); - %put &sysmacroname: SYSMSG= %sysfunc(sysmsg()); - %let rc=%sysfunc(close(&dsid)); -%end; -/* send them out without spaces or quote markers */ -%do;%unquote(&out)%end; -%mend mf_getfmtlist; diff --git a/009_macros/mf_getfmtname.sas b/009_macros/mf_getfmtname.sas deleted file mode 100644 index 1998f5b..0000000 --- a/009_macros/mf_getfmtname.sas +++ /dev/null @@ -1,44 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extracts a format name from a fully defined format - @details Converts formats in like $thi3. and th13.2 $THI and TH. - Usage: - - %put %mf_getfmtname(8.); - %put %mf_getfmtname($4.); - %put %mf_getfmtname(comma14.10); - - Returns: - - > W - > $CHAR - > COMMA - - Note that system defaults are inferred from the values provided. - - @param [in] fmt The fully defined format. If left blank, nothing is returned. - - @returns The name (without width or decimal) of the format. - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_getfmtname(fmt -)/*/STORE SOURCE*/ /minoperator mindelimiter=' '; - -%local out dsid nvars x rc fmt; - -/* extract actual format name from the format definition */ -%let fmt=%scan(&fmt,1,.); -%do %while(%substr(&fmt,%length(&fmt),1) in 1 2 3 4 5 6 7 8 9 0); - %if %length(&fmt)=1 %then %let fmt=W; - %else %let fmt=%substr(&fmt,1,%length(&fmt)-1); -%end; - -%if &fmt=$ %then %let fmt=$CHAR; - -/* send them out without spaces or quote markers */ -%do;%unquote(%upcase(&fmt))%end; -%mend mf_getfmtname; diff --git a/009_macros/mf_getgitbranch.sas b/009_macros/mf_getgitbranch.sas deleted file mode 100644 index 486d8d6..0000000 --- a/009_macros/mf_getgitbranch.sas +++ /dev/null @@ -1,37 +0,0 @@ -/*** HELP START ***//** - @file - @brief Retrieves the current branch from a local GIT repo - @details In a local git repository, the current branch is always available in - the `.git/HEAD` file in a format like this: `ref: refs/heads/master` - - This macro simply reads the file and returns the last word (eg `master`). - - Example usage: - - %let gitdir=%sysfunc(pathname(work))/core; - %let repo=https://github.com/sasjs/core; - %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&gitdir)); - - %put The current branch is %mf_getgitbranch(&gitdir); - - @param [in] gitdir The directory containing the GIT repository - -

SAS Macros

- @li mf_readfile.sas - -

Related Macros

- @li mp_gitadd.sas - @li mp_gitlog.sas - @li mp_gitreleaseinfo.sas - @li mp_gitstatus.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getgitbranch(gitdir -)/*/STORE SOURCE*/; - - %scan(%mf_readfile(&gitdir/.git/HEAD),-1) - -%mend mf_getgitbranch; diff --git a/009_macros/mf_getkeyvalue.sas b/009_macros/mf_getkeyvalue.sas deleted file mode 100644 index 59fb355..0000000 --- a/009_macros/mf_getkeyvalue.sas +++ /dev/null @@ -1,33 +0,0 @@ -/*** HELP START ***//** - @file - @brief retrieves a key value pair from a control dataset - @details By default, control dataset is work.mp_setkeyvalue. Usage: - - %mp_setkeyvalue(someindex,22,type=N) - %put %mf_getkeyvalue(someindex) - - - @param [in] key Provide a key on which to perform the lookup - @param [in] libds= (work.mp_setkeyvalue) The library.dataset which holds the - parameters - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getkeyvalue(key,libds=work.mp_setkeyvalue -)/*/STORE SOURCE*/; -%local ds dsid key valc valn type rc; -%let dsid=%sysfunc(open(&libds(where=(key="&key")))); -%syscall set(dsid); -%let rc = %sysfunc(fetch(&dsid)); -%let rc = %sysfunc(close(&dsid)); - -%if &type=N %then %do; - &valn -%end; -%else %if &type=C %then %do; - &valc -%end; -%else %put %str(ERR)OR: Unable to find key &key in ds &libds; -%mend mf_getkeyvalue; diff --git a/009_macros/mf_getplatform.sas b/009_macros/mf_getplatform.sas deleted file mode 100644 index ec5d62a..0000000 --- a/009_macros/mf_getplatform.sas +++ /dev/null @@ -1,75 +0,0 @@ -/*** HELP START ***//** - @file mf_getplatform.sas - @brief Returns platform specific variables - @details Enables platform specific variables to be returned - - %put %mf_getplatform(); - - returns one of: - - @li SASMETA - @li SASVIYA - @li SASJS - @li BASESAS - - @param [in] switch the param for which to return a platform specific variable - -

SAS Macros

- @li mf_mval.sas - @li mf_trimstr.sas - - @version 9.4 / 3.4 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getplatform(switch -)/*/STORE SOURCE*/; -%local a b c; -%if &switch.NONE=NONE %then %do; - %if %symexist(sasjsprocessmode) %then %do; - %if &sasjsprocessmode=Stored Program %then %do; - SASJS - %return; - %end; - %end; - %if %symexist(sysprocessmode) %then %do; - %if "&sysprocessmode"="SAS Object Server" - or "&sysprocessmode"= "SAS Compute Server" %then %do; - SASVIYA - %end; - %else %if "&sysprocessmode"="SAS Stored Process Server" - or "&sysprocessmode"="SAS Workspace Server" - %then %do; - SASMETA - %return; - %end; - %else %do; - BASESAS - %return; - %end; - %end; - %else %if %symexist(_metaport) or %symexist(_metauser) %then %do; - SASMETA - %return; - %end; - %else %do; - BASESAS - %return; - %end; -%end; -%else %if &switch=SASSTUDIO %then %do; - /* return the version of SAS Studio else 0 */ - %if %mf_mval(_CLIENTAPP)=%str(SAS Studio) %then %do; - %let a=%mf_mval(_CLIENTVERSION); - %let b=%scan(&a,1,.); - %if %eval(&b >2) %then %do; - &b - %end; - %else 0; - %end; - %else 0; -%end; -%else %if &switch=VIYARESTAPI %then %do; - %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/) -%end; -%mend mf_getplatform; diff --git a/009_macros/mf_getquotedstr.sas b/009_macros/mf_getquotedstr.sas deleted file mode 100644 index 98ab31b..0000000 --- a/009_macros/mf_getquotedstr.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief Adds custom quotes / delimiters to a delimited string - @details Can be used in open code, eg as follows: - - %put %mf_getquotedstr(blah blah blah); - - which returns: -> 'blah','blah','blah' - - Alternatively: - - %put %mf_getquotedstr(these words are double quoted,quote=D) - - for: -> "these","words","are","double","quoted" - - @param [in] in_str The unquoted, spaced delimited string to transform - @param [in] dlm= (,) The delimeter to be applied to the output (default comma) - @param [in] indlm= ( ) The delimeter used for the input (default is space) - @param [in] quote= (S) The quote mark to apply (S=Single, D=Double, N=None). - If any other value than uppercase S or D is supplied, then that value will - be used as the quoting character. - @return output returns a string with the newly quoted / delimited output. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - - -%macro mf_getquotedstr(IN_STR - ,DLM=%str(,) - ,QUOTE=S - ,indlm=%str( ) -)/*/STORE SOURCE*/; - /* credit Rowland Hale - byte34 is double quote, 39 is single quote */ - %if "e=S %then %let quote=%qsysfunc(byte(39)); - %else %if "e=D %then %let quote=%qsysfunc(byte(34)); - %else %if "e=N %then %let quote=; - %local i item buffer; - %let i=1; - %do %while (%qscan(&IN_STR,&i,%str(&indlm)) ne %str() ) ; - %let item=%qscan(&IN_STR,&i,%str(&indlm)); - %if %bquote("E) ne %then %let item="E%qtrim(&item)"E; - %else %let item=%qtrim(&item); - - %if (&i = 1) %then %let buffer =%qtrim(&item); - %else %let buffer =&buffer&DLM%qtrim(&item); - - %let i = %eval(&i+1); - %end; - - %let buffer=%sysfunc(coalescec(%qtrim(&buffer),"E"E)); - - &buffer - -%mend mf_getquotedstr; diff --git a/009_macros/mf_getschema.sas b/009_macros/mf_getschema.sas deleted file mode 100644 index fec8f88..0000000 --- a/009_macros/mf_getschema.sas +++ /dev/null @@ -1,43 +0,0 @@ -/*** HELP START ***//** - @file mf_getschema.sas - @brief Returns the database schema of a SAS library - @details Usage: - - %put %mf_getschema(MYDB); - - returns: - > dbo - - @param [in] libref Library reference (also accepts a 2 level libds ref). - - @return output returns the library schema for the FIRST library encountered - - @warning will only return the FIRST library schema - for concatenated - libraries, with different schemas, inconsistent results may be encountered. - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mf_getschema(libref -)/*/STORE SOURCE*/; - %local dsid vnum rc schema; - /* in case the parameter is a libref.tablename, pull off just the libref */ - %let libref = %upcase(%scan(&libref, 1, %str(.))); - %let dsid=%sysfunc(open(sashelp.vlibnam(where=( - libname="%upcase(&libref)" and sysname='Schema/Owner' - )),i)); - %if (&dsid ^= 0) %then %do; - %let vnum=%sysfunc(varnum(&dsid,SYSVALUE)); - %let rc=%sysfunc(fetch(&dsid)); - %let schema=%sysfunc(getvarc(&dsid,&vnum)); - %put &libref. schema is &schema.; - %let rc= %sysfunc(close(&dsid)); - %end; - - &schema - -%mend mf_getschema; - -/** @endcond */ diff --git a/009_macros/mf_getuniquefileref.sas b/009_macros/mf_getuniquefileref.sas deleted file mode 100644 index de84935..0000000 --- a/009_macros/mf_getuniquefileref.sas +++ /dev/null @@ -1,60 +0,0 @@ -/*** HELP START ***//** - @file - @brief Assigns and returns an unused fileref - @details Using the native approach for assigning filerefs fails as some - procedures (such as proc http) do not recognise the temporary names (starting - with a hash), returning a message such as: - - > ERROR 22-322: Expecting a name. - - This macro works by attempting a random fileref (with a prefix), seeing if it - is already assigned, and if not - returning the fileref. - - If your process can accept filerefs with the hash (#) prefix, then set - `prefix=0` to revert to the native approach - which is significantly faster - when there are a lot of filerefs in a session. - - Use as follows: - - %let fileref1=%mf_getuniquefileref(); - %let fileref2=%mf_getuniquefileref(prefix=0); - %put &fileref1 &fileref2; - - which returns filerefs similar to: - -> _7432233 #LN00070 - - @param [in] prefix= (_) first part of fileref. Remember that filerefs can only - be 8 characters, so a 7 letter prefix would mean `maxtries` should be 10. - if using zero (0) as the prefix, a native assignment is used. - @param [in] maxtries= (1000) the last part of the libref. Must be an integer. - @param [in] lrecl= (32767) Provide a default lrecl with which to initialise - the generated fileref. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getuniquefileref(prefix=_,maxtries=1000,lrecl=32767); - %local rc fname; - %if &prefix=0 %then %do; - %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl)); - %if &rc %then %put %sysfunc(sysmsg()); - &fname - %end; - %else %do; - %local x len; - %let len=%eval(8-%length(&prefix)); - %let x=0; - %do x=0 %to &maxtries; - %let fname=&prefix%substr(%sysfunc(ranuni(0)),3,&len); - %if %sysfunc(fileref(&fname)) > 0 %then %do; - %let rc=%sysfunc(filename(fname,,temp,lrecl=&lrecl)); - %if &rc %then %put %sysfunc(sysmsg()); - &fname - %return; - %end; - %end; - %put unable to find available fileref after &maxtries attempts; - %end; -%mend mf_getuniquefileref; diff --git a/009_macros/mf_getuniquelibref.sas b/009_macros/mf_getuniquelibref.sas deleted file mode 100644 index 2524ce2..0000000 --- a/009_macros/mf_getuniquelibref.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns an unused libref - @details Use as follows: - - libname mclib0 (work); - libname mclib1 (work); - libname mclib2 (work); - - %let libref=%mf_getuniquelibref(); - %put &=libref; - - which returns: - -> mclib3 - - A blank value is returned if no usable libname is determined. - - @param [in] prefix= (mclib) first part of the returned libref. As librefs can - be as long as 8 characters, a maximum length of 7 characters is premitted - for this prefix. - @param [in] maxtries= (1000) Deprecated parameter. Remains here to ensure a - non-breaking change. Will be removed in v5. - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getuniquelibref(prefix=mclib,maxtries=1000); - %local x; - - %if ( %length(&prefix) gt 7 ) %then %do; - %put %str(ERR)OR: The prefix parameter cannot exceed 7 characters.; - 0 - %return; - %end; - %else %if (%sysfunc(NVALID(&prefix,v7))=0) %then %do; - %put %str(ERR)OR: Invalid prefix (&prefix); - 0 - %return; - %end; - - /* Set maxtries equal to '10 to the power of [# unused characters] - 1' */ - %let maxtries=%eval(10**(8-%length(&prefix))-1); - - %do x = 0 %to &maxtries; - %if %sysfunc(libref(&prefix&x)) ne 0 %then %do; - &prefix&x - %return; - %end; - %let x = %eval(&x + 1); - %end; - - %put %str(ERR)OR: No usable libref in range &prefix.0-&maxtries; - %put %str(ERR)OR- Try reducing the prefix or deleting some libraries!; - 0 -%mend mf_getuniquelibref; diff --git a/009_macros/mf_getuniquename.sas b/009_macros/mf_getuniquename.sas deleted file mode 100644 index 6df6bff..0000000 --- a/009_macros/mf_getuniquename.sas +++ /dev/null @@ -1,22 +0,0 @@ -/*** HELP START ***//** - @file mf_getuniquename.sas - @brief Returns a shortened (32 char) GUID as a valid SAS name - @details Use as follows: - - %let myds=%mf_getuniquename(); - %put &=myds; - - which returns: - -> MCc59c750610321d4c8bf75faadbcd22 - - @param [in] prefix= (MC) Sets a prefix for the new name - - @version 9.3 - @author Allan Bowe -**//*** HELP END ***/ - - -%macro mf_getuniquename(prefix=MC); -&prefix.%substr(%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32-%length(&prefix)) -%mend mf_getuniquename; diff --git a/009_macros/mf_getuser.sas b/009_macros/mf_getuser.sas deleted file mode 100644 index 64b44fc..0000000 --- a/009_macros/mf_getuser.sas +++ /dev/null @@ -1,42 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a userid according to session context - @details In a workspace session, a user is generally represented by - &sysuserid or SYS_COMPUTE_SESSION_OWNER if it exists. - In a Stored Process session, &sysuserid - resolves to a system account (default=sassrv) and instead there are several - metadata username variables to choose from (_metauser, _metaperson - ,_username, _secureusername). The OS account is represented by - _secureusername whilst the metadata account is under - _metaperson. - - %let user= %mf_getUser(); - %put &user; - - @return SYSUSERID (if workspace server) - @return _METAPERSON (if stored process server) - @return SYS_COMPUTE_SESSION_OWNER (if Viya compute session) - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getuser( -)/*/STORE SOURCE*/; - %local user; - - %if %symexist(_sasjs_username) %then %let user=&_sasjs_username; - %else %if %symexist(SYS_COMPUTE_SESSION_OWNER) %then %do; - %let user=&SYS_COMPUTE_SESSION_OWNER; - %end; - %else %if %symexist(_metaperson) %then %do; - %if %length(&_metaperson)=0 %then %let user=&sysuserid; - /* sometimes SAS will add @domain extension - remove for consistency */ - /* but be sure to quote in case of usernames with commas */ - %else %let user=%unquote(%scan(%quote(&_metaperson),1,@)); - %end; - %else %let user=&sysuserid; - - %quote(&user) - -%mend mf_getuser; diff --git a/009_macros/mf_getvalue.sas b/009_macros/mf_getvalue.sas deleted file mode 100644 index 6239126..0000000 --- a/009_macros/mf_getvalue.sas +++ /dev/null @@ -1,36 +0,0 @@ -/*** HELP START ***//** - @file - @brief Retrieves a value from a dataset. If no filter supplied, then first - record is used. - @details Be sure to %quote() your where clause. Example usage: - - %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15)); - %put %mf_getvalue(sashelp.class,name); - -

SAS Macros

- @li mf_getattrn.sas - -

Related Macros

- @li mp_setkeyvalue.sas - - @param [in] libds dataset to query - @param [in] variable the variable which contains the value to return. - @param [in] filter= (1) contents of where clause - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_getvalue(libds,variable,filter=1 -)/*/STORE SOURCE*/; - %if %mf_getattrn(&libds,NLOBS)>0 %then %do; - %local dsid rc &variable; - %let dsid=%sysfunc(open(&libds(where=(&filter)))); - %syscall set(dsid); - %let rc = %sysfunc(fetch(&dsid)); - %let rc = %sysfunc(close(&dsid)); - - %trim(&&&variable) - - %end; -%mend mf_getvalue; diff --git a/009_macros/mf_getvarcount.sas b/009_macros/mf_getvarcount.sas deleted file mode 100644 index 8b1a8f7..0000000 --- a/009_macros/mf_getvarcount.sas +++ /dev/null @@ -1,49 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns number of variables in a dataset - @details Useful to identify those renagade datasets that have no columns! - Can also be used to count for numeric, or character columns - - %put Number of Variables=%mf_getvarcount(sashelp.class); - %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C); - %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N); - - returns: - > Number of Variables=4 - - - @param [in] libds Two part dataset (or view) reference. - @param [in] typefilter= (A) Filter for certain types of column. Valid values: - @li A Count All columns - @li C Count Character columns only - @li N Count Numeric columns only - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_getvarcount(libds,typefilter=A -)/*/STORE SOURCE*/; - %local dsid nvars rc outcnt x; - %let dsid=%sysfunc(open(&libds)); - %let nvars=.; - %let outcnt=0; - %let typefilter=%upcase(&typefilter); - %if &dsid %then %do; - %let nvars=%sysfunc(attrn(&dsid,NVARS)); - %if &typefilter=A %then %let outcnt=&nvars; - %else %if &nvars>0 %then %do x=1 %to &nvars; - /* increment based on variable type */ - %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do; - %let outcnt=%eval(&outcnt+1); - %end; - %end; - %let rc=%sysfunc(close(&dsid)); - %end; - %else %do; - %put unable to open &libds (rc=&dsid); - %let rc=%sysfunc(close(&dsid)); - %end; - &outcnt -%mend mf_getvarcount; diff --git a/009_macros/mf_getvarformat.sas b/009_macros/mf_getvarformat.sas deleted file mode 100644 index eb77b27..0000000 --- a/009_macros/mf_getvarformat.sas +++ /dev/null @@ -1,73 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the format of a variable - @details Uses varfmt function to identify the format of a particular variable. - Usage: - - data test; - format str1 $1. num1 datetime19.; - str2='hello mum!'; num2=666; - stop; - run; - %put %mf_getVarFormat(test,str1); - %put %mf_getVarFormat(work.test,num1); - %put %mf_getVarFormat(test,str2,force=1); - %put %mf_getVarFormat(work.test,num2,force=1); - %put %mf_getVarFormat(test,renegade); - - returns: - - $1. - DATETIME19. - $10. - 8. - NOTE: Variable renegade does not exist in test - - @param [in] libds Two part dataset (or view) reference. - @param [in] var Variable name for which a format should be returned - @param [in] force= (0) Set to 1 to supply a default if the variable has no - format - @returns outputs format - - @author Allan Bowe - @version 9.2 -**//*** HELP END ***/ - -%macro mf_getVarFormat(libds /* two level ds name */ - , var /* variable name from which to return the format */ - , force=0 -)/*/STORE SOURCE*/; - %local dsid vnum vformat rc vlen vtype; - /* Open dataset */ - %let dsid = %sysfunc(open(&libds)); - %if &dsid > 0 %then %do; - /* Get variable number */ - %let vnum = %sysfunc(varnum(&dsid, &var)); - /* Get variable format */ - %if(&vnum > 0) %then %let vformat=%sysfunc(varfmt(&dsid, &vnum)); - %else %do; - %put NOTE: Variable &var does not exist in &libds; - %let rc = %sysfunc(close(&dsid)); - %return; - %end; - %end; - %else %do; - %put &sysmacroname: dataset &libds not opened! (rc=&dsid); - %put &sysmacroname: %sysfunc(sysmsg()); - %return; - %end; - - /* supply a default if no format available */ - %if %length(&vformat)<2 & &force=1 %then %do; - %let vlen = %sysfunc(varlen(&dsid, &vnum)); - %let vtype = %sysfunc(vartype(&dsid, &vnum.)); - %if &vtype=C %then %let vformat=$&vlen..; - %else %let vformat=best.; - %end; - - - /* Close dataset */ - %let rc = %sysfunc(close(&dsid)); - /* Return variable format */ - &vformat -%mend mf_getVarFormat; diff --git a/009_macros/mf_getvarlen.sas b/009_macros/mf_getvarlen.sas deleted file mode 100644 index 05af96c..0000000 --- a/009_macros/mf_getvarlen.sas +++ /dev/null @@ -1,56 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the length of a variable - @details Uses varlen function to identify the length of a particular variable. - Usage: - - data test; - format str $1. num datetime19.; - stop; - run; - %put %mf_getVarLen(test,str); - %put %mf_getVarLen(work.test,num); - %put %mf_getVarLen(test,renegade); - - returns: - - 1 - 8 - NOTE: Variable renegade does not exist in test - - @param [in] libds Two part dataset (or view) reference. - @param [in] var Variable name for which a length should be returned - @returns outputs length - - @author Allan Bowe - @version 9.2 - -**//*** HELP END ***/ - -%macro mf_getVarLen(libds /* two level ds name */ - , var /* variable name from which to return the length */ -)/*/STORE SOURCE*/; - %local dsid vnum vlen rc; - /* Open dataset */ - %let dsid = %sysfunc(open(&libds)); - %if &dsid > 0 %then %do; - /* Get variable number */ - %let vnum = %sysfunc(varnum(&dsid, &var)); - /* Get variable format */ - %if(&vnum > 0) %then %let vlen = %sysfunc(varlen(&dsid, &vnum)); - %else %do; - %put NOTE: Variable &var does not exist in &libds; - %let vlen = %str( ); - %end; - %end; - %else %do; - %put &sysmacroname: dataset &libds not opened! (rc=&dsid); - %put &sysmacroname: %sysfunc(sysmsg()); - %return; - %end; - - /* Close dataset */ - %let rc = %sysfunc(close(&dsid)); - /* Return variable format */ - &vlen -%mend mf_getVarLen; diff --git a/009_macros/mf_getvarlist.sas b/009_macros/mf_getvarlist.sas deleted file mode 100644 index 428efeb..0000000 --- a/009_macros/mf_getvarlist.sas +++ /dev/null @@ -1,74 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns dataset variable list direct from header - @details WAY faster than dictionary tables or sas views, and can - also be called in macro logic (is pure macro). Can be used in open code, - eg as follows: - - %put List of Variables=%mf_getvarlist(sashelp.class); - - returns: - > List of Variables=Name Sex Age Height Weight - - For a seperated list of column values: - - %put %mf_getvarlist(sashelp.class,dlm=%str(,),quote=double); - - returns: - > "Name","Sex","Age","Height","Weight" - - @param [in] libds Two part dataset (or view) reference. - @param [in] dlm= ( ) Provide a delimiter (eg comma or space) to separate the - variables - @param [in] quote= (none) use either DOUBLE or SINGLE to quote the results - @param [in] typefilter= (A) Filter for certain types of column. Valid values: - @li A Return All columns - @li C Return Character columns - @li N Return Numeric columns - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_getvarlist(libds - ,dlm=%str( ) - ,quote=no - ,typefilter=A -)/*/STORE SOURCE*/; - /* declare local vars */ - %local outvar dsid nvars x rc dlm q var vtype; - - /* credit Rowland Hale - byte34 is double quote, 39 is single quote */ - %if %upcase("e)=DOUBLE %then %let q=%qsysfunc(byte(34)); - %else %if %upcase("e)=SINGLE %then %let q=%qsysfunc(byte(39)); - /* open dataset in macro */ - %let dsid=%sysfunc(open(&libds)); - - %if &dsid %then %do; - %let nvars=%sysfunc(attrn(&dsid,NVARS)); - %if &nvars>0 %then %do; - /* add variables with supplied delimeter */ - %do x=1 %to &nvars; - /* get variable type */ - %let vtype=%sysfunc(vartype(&dsid,&x)); - %if &vtype=&typefilter or &typefilter=A %then %do; - %let var=&q.%sysfunc(varname(&dsid,&x))&q.; - %if &var=&q&q %then %do; - %put &sysmacroname: Empty column found in &libds!; - %let var=&q. &q.; - %end; - %if %quote(&outvar)=%quote() %then %let outvar=&var; - %else %let outvar=&outvar.&dlm.&var.; - %end; - %end; - %end; - %let rc=%sysfunc(close(&dsid)); - %end; - %else %do; - %put &sysmacroname: Unable to open &libds (rc=&dsid); - %put &sysmacroname: SYSMSG= %sysfunc(sysmsg()); - %let rc=%sysfunc(close(&dsid)); - %end; - %do;%unquote(&outvar)%end; -%mend mf_getvarlist; diff --git a/009_macros/mf_getvarnum.sas b/009_macros/mf_getvarnum.sas deleted file mode 100644 index a2dd307..0000000 --- a/009_macros/mf_getvarnum.sas +++ /dev/null @@ -1,58 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the position of a variable in dataset (varnum attribute). - @details Uses varnum function to determine position. - -Usage: - - data work.test; - format str $1. num datetime19.; - stop; - run; - %put %mf_getVarNum(work.test,str); - %put %mf_getVarNum(work.test,num); - %put %mf_getVarNum(work.test,renegade); - -returns: - - > 1 - - > 2 - - > NOTE: Variable renegade does not exist in test - - @param [in] libds Two part dataset (or view) reference. - @param [in] var Variable name for which a position should be returned - - @author Allan Bowe - @version 9.2 - -**//*** HELP END ***/ - -%macro mf_getVarNum(libds /* two level ds name */ - , var /* variable name from which to return the format */ -)/*/STORE SOURCE*/; - %local dsid vnum rc; - /* Open dataset */ - %let dsid = %sysfunc(open(&libds)); - %if &dsid > 0 %then %do; - /* Get variable number */ - %let vnum = %sysfunc(varnum(&dsid, &var)); - %if(&vnum <= 0) %then %do; - %put NOTE: Variable &var does not exist in &libds; - %let vnum = %str( ); - %end; - %end; - %else %do; - %put &sysmacroname: dataset &libds not opened! (rc=&dsid); - %put &sysmacroname: %sysfunc(sysmsg()); - %return; - %end; - - /* Close dataset */ - %let rc = %sysfunc(close(&dsid)); - - /* Return variable number */ - &vnum. - -%mend mf_getVarNum; diff --git a/009_macros/mf_getvartype.sas b/009_macros/mf_getvartype.sas deleted file mode 100644 index 27b77d9..0000000 --- a/009_macros/mf_getvartype.sas +++ /dev/null @@ -1,52 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns variable type - Character (C) or Numeric (N) - @details -Usage: - - data test; - length str $1. num 8.; - stop; - run; - %put %mf_getvartype(test,str); - %put %mf_getvartype(work.test,num); - - - - @param [in] libds Two part dataset (or view) reference. - @param [in] var the variable name to be checked - @return output returns C or N depending on variable type. If variable - does not exist then a blank is returned and a note is written to the log. - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_getvartype(libds /* two level name */ - , var /* variable name from which to return the type */ -)/*/STORE SOURCE*/; - %local dsid vnum vtype rc; - /* Open dataset */ - %let dsid = %sysfunc(open(&libds)); - %if &dsid. > 0 %then %do; - /* Get variable number */ - %let vnum = %sysfunc(varnum(&dsid, &var)); - /* Get variable type (C/N) */ - %if(&vnum. > 0) %then %let vtype = %sysfunc(vartype(&dsid, &vnum.)); - %else %do; - %put NOTE: Variable &var does not exist in &libds; - %let vtype = %str( ); - %end; - %end; - %else %do; - %put &sysmacroname: dataset &libds not opened! (rc=&dsid); - %put &sysmacroname: %sysfunc(sysmsg()); - %return; - %end; - - /* Close dataset */ - %let rc = %sysfunc(close(&dsid)); - /* Return variable type */ - &vtype -%mend mf_getvartype; diff --git a/009_macros/mf_getxengine.sas b/009_macros/mf_getxengine.sas deleted file mode 100644 index b6c5044..0000000 --- a/009_macros/mf_getxengine.sas +++ /dev/null @@ -1,43 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns the engine type of a SAS fileref - @details Queries sashelp.vextfl to get the xengine value. - Usage: - - filename feng temp; - %put %mf_getxengine(feng); - - returns: - > TEMP - - @param [in] fref The fileref to check - - @returns The XENGINE value in sashelp.vextfl or 0 if not found. - - @version 9.2 - @author Allan Bowe - -

Related Macros

- @li mf_getengine.sas - -**//*** HELP END ***/ - -%macro mf_getxengine(fref -)/*/STORE SOURCE*/; - %local dsid engnum rc engine; - - %let dsid=%sysfunc( - open(sashelp.vextfl(where=(fileref="%upcase(&fref)")),i) - ); - %if (&dsid ^= 0) %then %do; - %let engnum=%sysfunc(varnum(&dsid,XENGINE)); - %let rc=%sysfunc(fetch(&dsid)); - %let engine=%sysfunc(getvarc(&dsid,&engnum)); - %* put &fref. ENGINE is &engine.; - %let rc= %sysfunc(close(&dsid)); - %end; - %else %let engine=0; - - &engine - -%mend mf_getxengine; diff --git a/009_macros/mf_increment.sas b/009_macros/mf_increment.sas deleted file mode 100644 index 77d4f16..0000000 --- a/009_macros/mf_increment.sas +++ /dev/null @@ -1,29 +0,0 @@ -/*** HELP START ***//** - @file - @brief Increments a macro variable - @details Useful outside of do-loops - will increment a macro variable every - time it is called. - - Example: - - %let cnt=1; - %put We have run %mf_increment(cnt) lines; - %put Now we have run %mf_increment(cnt) lines; - %put There are %mf_increment(cnt) lines in total; - - @param [in] macro_name The name of the macro variable to increment - @param [in] incr= (1) The amount to add or subtract to the macro - -

Related Files

- @li mf_increment.test.sas - -**//*** HELP END ***/ - -%macro mf_increment(macro_name,incr=1); - - /* iterate the value */ - %let ¯o_name=%eval(&&¯o_name+&incr); - /* return the value */ - &&¯o_name - -%mend mf_increment; diff --git a/009_macros/mf_isdir.sas b/009_macros/mf_isdir.sas deleted file mode 100644 index f0633e0..0000000 --- a/009_macros/mf_isdir.sas +++ /dev/null @@ -1,34 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a path is a valid directory - @details - Usage: - - %let isdir=%mf_isdir(/tmp); - - With thanks and full credit to Andrea Defronzo - - https://www.linkedin.com/in/andrea-defronzo-b1a47460/ - - @param [in] path Full path of the file/directory to be checked - - @return output returns 1 if path is a directory, 0 if it is not - - @version 9.2 -**//*** HELP END ***/ - -%macro mf_isdir(path -)/*/STORE SOURCE*/; - %local rc did is_directory fref_t; - - %let is_directory = 0; - %let rc = %sysfunc(filename(fref_t, %superq(path))); - %let did = %sysfunc(dopen(&fref_t.)); - %if &did. ^= 0 %then %do; - %let is_directory = 1; - %let rc = %sysfunc(dclose(&did.)); - %end; - %let rc = %sysfunc(filename(fref_t)); - - &is_directory - -%mend mf_isdir; diff --git a/009_macros/mf_isint.sas b/009_macros/mf_isint.sas deleted file mode 100644 index 002b3ee..0000000 --- a/009_macros/mf_isint.sas +++ /dev/null @@ -1,36 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns 1 if the variable contains only digits 0-9, else 0 - @details Note that numerics containing any punctuation (including decimals - or exponents) will be flagged zero. - - If you'd like support for this, then do raise an issue (or even better, a - pull request!) - - Usage: - - %put %mf_isint(1) returns 1; - %put %mf_isint(1.1) returns 0; - %put %mf_isint(%str(1,1)) returns 0; - - @param [in] arg input value to check - - @version 9.2 -**//*** HELP END ***/ - -%macro mf_isint(arg -)/*/STORE SOURCE*/; - - /* blank val is not an integer */ - %if "&arg"="" %then %do;0%return;%end; - - /* remove minus sign if exists */ - %local val; - %if "%substr(%str(&arg),1,1)"="-" %then %let val=%substr(%str(&arg),2); - %else %let val=&arg; - - /* check remaining chars */ - %if %sysfunc(findc(%str(&val),,kd)) %then %do;0%end; - %else %do;1%end; - -%mend mf_isint; diff --git a/009_macros/mf_islibds.sas b/009_macros/mf_islibds.sas deleted file mode 100644 index 5ec382a..0000000 --- a/009_macros/mf_islibds.sas +++ /dev/null @@ -1,40 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks whether a string follows correct library.dataset format - @details Many macros in the core library accept a library.dataset parameter - referred to as 'libds'. This macro validates the structure of that parameter, - eg: - - @li 8 character libref? - @li 32 character dataset? - @li contains a period? - - It does NOT check whether the dataset exists, or if the library is assigned. - - Usage: - - %put %mf_islibds(work.something)=1; - %put %mf_islibds(nolib)=0; - %put %mf_islibds(badlibref.ds)=0; - %put %mf_islibds(w.t.f)=0; - - @param [in] libds The string to be checked - - @return output Returns 1 if libds is valid, 0 if it is not - -

Related Macros

- @li mf_islibds.test.sas - @li mp_validatecol.sas - - @version 9.2 -**//*** HELP END ***/ - -%macro mf_islibds(libds -)/*/STORE SOURCE*/; - -%local regex; -%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i))); - -%sysfunc(prxmatch(®ex,&libds)) - -%mend mf_islibds; diff --git a/009_macros/mf_loc.sas b/009_macros/mf_loc.sas deleted file mode 100644 index caa354b..0000000 --- a/009_macros/mf_loc.sas +++ /dev/null @@ -1,34 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns physical location of various SAS items - @details Returns location of the PlatformObjectFramework tools - Usage: - - %put %mf_loc(POF); %*location of PlatformObjectFramework tools; - - @param [in] loc The item to locate, eg: - @li PLAATFORMOBJECTFRAMEWORK (or POF) - @li VIYACONFG - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_loc(loc); -%let loc=%upcase(&loc); -%local root; - -%if &loc=POF or &loc=PLATFORMOBJECTFRAMEWORK %then %do; - %let root=%sysget(SASROOT); - %let root=%substr(&root,1,%index(&root,SASFoundation)-2); - %let root=&root/SASPlatformObjectFramework/&sysver; - %put Batch tools located at: &root; - &root -%end; -%else %if &loc=VIYACONFIG %then %do; - %let root=/opt/sas/viya/config; - %put Viya Config located at: &root; - &root -%end; - -%mend mf_loc; diff --git a/009_macros/mf_mkdir.sas b/009_macros/mf_mkdir.sas deleted file mode 100644 index 860537f..0000000 --- a/009_macros/mf_mkdir.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a directory, including any intermediate directories - @details Works on windows and unix environments via dcreate function. -Usage: - - %mf_mkdir(/some/path/name) - - - @param [in] dir Relative or absolute pathname. Unquoted. - @version 9.2 - -**//*** HELP END ***/ - -%macro mf_mkdir(dir -)/*/STORE SOURCE*/; - - %local lastchar child parent; - - %let lastchar = %substr(&dir, %length(&dir)); - %if (%bquote(&lastchar) eq %str(:)) %then %do; - /* Cannot create drive mappings */ - %return; - %end; - - %if (%bquote(&lastchar)=%str(/)) or (%bquote(&lastchar)=%str(\)) %then %do; - /* last char is a slash */ - %if (%length(&dir) eq 1) %then %do; - /* one single slash - root location is assumed to exist */ - %return; - %end; - %else %do; - /* strip last slash */ - %let dir = %substr(&dir, 1, %length(&dir)-1); - %end; - %end; - - %if (%sysfunc(fileexist(%bquote(&dir))) = 0) %then %do; - /* directory does not exist so prepare to create */ - /* first get the childmost directory */ - %let child = %scan(&dir, -1, %str(/\:)); - - /* - If child name = path name then there are no parents to create. Else - they must be recursively scanned. - */ - - %if (%length(&dir) gt %length(&child)) %then %do; - %let parent = %substr(&dir, 1, %length(&dir)-%length(&child)); - %mf_mkdir(&parent) - %end; - - /* - Now create the directory. Complain loudly of any errs. - */ - - %let dname = %sysfunc(dcreate(&child, &parent)); - %if (%bquote(&dname) eq ) %then %do; - %put %str(ERR)OR: could not create &parent + &child; - %abort cancel; - %end; - %else %do; - %put Directory created: &dir; - %end; - %end; - /* exit quietly if directory did exist.*/ -%mend mf_mkdir; diff --git a/009_macros/mf_mval.sas b/009_macros/mf_mval.sas deleted file mode 100644 index b2ed792..0000000 --- a/009_macros/mf_mval.sas +++ /dev/null @@ -1,21 +0,0 @@ -/*** HELP START ***//** - @file mf_mval.sas - @brief Returns a macro variable value if the variable exists - @details - Use this macro to avoid repetitive use of `%if %symexist(MACVAR) %then` - type logic. - Usage: - - %if %mf_mval(maynotexist)=itdid %then %do; - - @param [in] var The macro variable NAME to return the (possible) value for - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mf_mval(var); - %if %symexist(&var) %then %do; - %superq(&var) - %end; -%mend mf_mval; diff --git a/009_macros/mf_nobs.sas b/009_macros/mf_nobs.sas deleted file mode 100644 index 2f2ab25..0000000 --- a/009_macros/mf_nobs.sas +++ /dev/null @@ -1,26 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns number of logical (undeleted) observations. - @details Beware - will not work on external database tables! - Is just a convenience macro for calling %mf_getattrn(). - - %put Number of observations=%mf_nobs(sashelp.class); - -

SAS Macros

- @li mf_getattrn.sas - - @param [in] libds library.dataset - - @return output returns result of the attrn value supplied, or log message - if err. - - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_nobs(libds -)/*/STORE SOURCE*/; - %mf_getattrn(&libds,NLOBS) -%mend mf_nobs; diff --git a/009_macros/mf_readfile.sas b/009_macros/mf_readfile.sas deleted file mode 100644 index 0b3ff48..0000000 --- a/009_macros/mf_readfile.sas +++ /dev/null @@ -1,63 +0,0 @@ -/*** HELP START ***//** - @file - @brief Reads the first line of a file using pure macro - @details Reads the first line of a file and returns it. Future versions may - read each line into a macro variable array. - - Generally, reading data into macro variables is not great as certain - nonprintable characters (such as CR, LF) may be dropped in the conversion. - - Usage: - - %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content) - - %put %mf_readfile(&sasjswork/myfile.txt); - - - @param [in] fpath Full path to file to be read - -

Related Macros

- @li mf_deletefile.sas - @li mf_writefile.sas - @li mf_readfile.test.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ - -%macro mf_readfile(fpath -)/*/STORE SOURCE*/; -%local fref rc fid fcontent; - -/* check file exists */ -%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do; - %put &=fref &=fpath; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%let fid=%sysfunc(fopen(&fref,I)); - -%if &fid=0 %then %do; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%if %sysfunc(fread(&fid)) = 0 %then %do; - %let rc=%sysfunc(fget(&fid,fcontent,65534)); - &fcontent -%end; - -/* -%do %while(%sysfunc(fread(&fid)) = 0); - %let rc=%sysfunc(fget(&fid,fcontent,65534)); - &fcontent -%end; -*/ - -%let rc=%sysfunc(fclose(&fid)); -%let rc=%sysfunc(filename(&fref)); - -%mend mf_readfile; -/** @endcond */ diff --git a/009_macros/mf_trimstr.sas b/009_macros/mf_trimstr.sas deleted file mode 100644 index 3e2a2de..0000000 --- a/009_macros/mf_trimstr.sas +++ /dev/null @@ -1,50 +0,0 @@ -/*** HELP START ***//** - @file mf_trimstr.sas - @brief Removes character(s) from the end, if they exist - @details If the designated characters exist at the end of the string, they - are removed - - %put %mf_trimstr(/blah/,/); * /blah; - %put %mf_trimstr(/blah/,h); * /blah/; - %put %mf_trimstr(/blah/,h/);* /bla; - -

SAS Macros

- - - @param [in] basestr The string to be modified - @param [in] trimstr The string to be removed from the end of `basestr`, if it - exists - - @return output returns result with the value of `trimstr` removed from the end - - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_trimstr(basestr,trimstr); -%local baselen trimlen trimval; - -/* return if basestr is shorter than trimstr (or 0) */ -%let baselen=%length(%superq(basestr)); -%let trimlen=%length(%superq(trimstr)); -%if &baselen < &trimlen or &baselen=0 %then %return; - -/* obtain the characters from the end of basestr */ -%let trimval=%qsubstr(%superq(basestr) - ,%length(%superq(basestr))-&trimlen+1 - ,&trimlen); - -/* compare and if matching, chop it off! */ -%if %superq(basestr)=%superq(trimstr) %then %do; - %return; -%end; -%else %if %superq(trimval)=%superq(trimstr) %then %do; - %qsubstr(%superq(basestr),1,%length(%superq(basestr))-&trimlen) -%end; -%else %do; - &basestr -%end; - -%mend mf_trimstr; diff --git a/009_macros/mf_uid.sas b/009_macros/mf_uid.sas deleted file mode 100644 index 48a2177..0000000 --- a/009_macros/mf_uid.sas +++ /dev/null @@ -1,21 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a unique ID based on system time in friendly format - @details format = YYYYMMDD_HHMMSSmmm__<3randomDigits> - - %put %mf_uid(); - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_uid( -)/*/STORE SOURCE*/; - %local today now; - %let today=%sysfunc(today(),yymmddn8.); - %let now=%sysfunc(compress(%sysfunc(time(),tod12.3),:.)); - - &today._&now._&sysjobid._%sysevalf(%sysfunc(ranuni(0))*999,CEIL) - -%mend mf_uid; diff --git a/009_macros/mf_verifymacvars.sas b/009_macros/mf_verifymacvars.sas deleted file mode 100644 index c22aa13..0000000 --- a/009_macros/mf_verifymacvars.sas +++ /dev/null @@ -1,72 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks if a set of macro variables exist AND contain values. - @details Writes ERROR to log if abortType is SOFT, else will call %mf_abort. - Usage: - - %let var1=x; - %let var2=y; - %put %mf_verifymacvars(var1 var2); - - Returns: - > 1 - -

SAS Macros

- @li mf_abort.sas - - @param [in] verifyvars Space separated list of macro variable names - @param [in] makeupcase= (NO) Set to YES to convert all variable VALUES to - uppercase. - @param [in] mAbort= (SOFT) Abort Type. When SOFT, simply writes an err - message to the log. - Set to any other value to call mf_abort (which can be configured to abort in - various fashions according to context). - - @warning will not be able to verify the following variables due to - naming clash! - - verifyVars - - verifyVar - - verifyIterator - - makeUpcase - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - - -%macro mf_verifymacvars( - verifyVars /* list of macro variable NAMES */ - ,makeUpcase=NO /* set to YES to make all the variable VALUES uppercase */ - ,mAbort=SOFT -)/*/STORE SOURCE*/; - - %local verifyIterator verifyVar abortmsg; - %do verifyIterator=1 %to %sysfunc(countw(&verifyVars,%str( ))); - %let verifyVar=%qscan(&verifyVars,&verifyIterator,%str( )); - %if not %symexist(&verifyvar) %then %do; - %let abortmsg= Variable &verifyVar is MISSING; - %goto exit_err; - %end; - %if %length(%trim(&&&verifyVar))=0 %then %do; - %let abortmsg= Variable &verifyVar is EMPTY; - %goto exit_err; - %end; - %if &makeupcase=YES %then %do; - %let &verifyVar=%upcase(&&&verifyvar); - %end; - %end; - - %goto exit_success; - %exit_err: - %put &abortmsg; - %mf_abort(iftrue=(&mabort ne SOFT), - mac=mf_verifymacvars, - msg=%str(&abortmsg) - ) - 0 - %return; - %exit_success: - 1 - -%mend mf_verifymacvars; diff --git a/009_macros/mf_wordsinstr1andstr2.sas b/009_macros/mf_wordsinstr1andstr2.sas deleted file mode 100644 index 44f2f49..0000000 --- a/009_macros/mf_wordsinstr1andstr2.sas +++ /dev/null @@ -1,53 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns words that are in both string 1 and string 2 - @details Compares two space separated strings and returns the words that are - in both. - Usage: - - %put %mf_wordsInStr1andStr2( - Str1=blah sss blaaah brah bram boo - ,Str2= blah blaaah brah ssss - ); - - returns: - > blah blaaah brah - - @param [in] str1= () string containing words to extract - @param [in] str2= () used to compare with the extract string - - @warning CASE SENSITIVE! - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_wordsInStr1andStr2( - Str1= /* string containing words to extract */ - ,Str2= /* used to compare with the extract string */ -)/*/STORE SOURCE*/; - -%local count_base count_extr i i2 extr_word base_word match outvar; -%if %length(&str1)=0 or %length(&str2)=0 %then %do; - %put base string (str1)= &str1; - %put compare string (str2) = &str2; - %return; -%end; -%let count_base=%sysfunc(countw(&Str2)); -%let count_extr=%sysfunc(countw(&Str1)); - -%do i=1 %to &count_extr; - %let extr_word=%scan(&Str1,&i,%str( )); - %let match=0; - %do i2=1 %to &count_base; - %let base_word=%scan(&Str2,&i2,%str( )); - %if &extr_word=&base_word %then %let match=1; - %end; - %if &match=1 %then %let outvar=&outvar &extr_word; -%end; - - &outvar - -%mend mf_wordsInStr1andStr2; - diff --git a/009_macros/mf_wordsinstr1butnotstr2.sas b/009_macros/mf_wordsinstr1butnotstr2.sas deleted file mode 100644 index 5d665f2..0000000 --- a/009_macros/mf_wordsinstr1butnotstr2.sas +++ /dev/null @@ -1,54 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns words that are in string 1 but not in string 2 - @details Compares two space separated strings and returns the words that are - in the first but not in the second. - - Note - case sensitive! - - Usage: - - %let x= %mf_wordsInStr1ButNotStr2( - Str1=blah sss blaaah brah bram boo - ,Str2= blah blaaah brah ssss - ); - - returns: - > sss bram boo - - @param [in] str1= () String containing words to extract - @param [in] str2= () Used to compare with the extract string - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mf_wordsInStr1ButNotStr2( - Str1= /* string containing words to extract */ - ,Str2= /* used to compare with the extract string */ -)/*/STORE SOURCE*/; - -%local count_base count_extr i i2 extr_word base_word match outvar; -%if %length(&str1)=0 or %length(&str2)=0 %then %do; - %put base string (str1)= &str1; - %put compare string (str2) = &str2; - %return; -%end; -%let count_base=%sysfunc(countw(&Str2)); -%let count_extr=%sysfunc(countw(&Str1)); - -%do i=1 %to &count_extr; - %let extr_word=%scan(&Str1,&i,%str( )); - %let match=0; - %do i2=1 %to &count_base; - %let base_word=%scan(&Str2,&i2,%str( )); - %if &extr_word=&base_word %then %let match=1; - %end; - %if &match=0 %then %let outvar=&outvar &extr_word; -%end; - - &outvar - -%mend mf_wordsInStr1ButNotStr2; - diff --git a/009_macros/mf_writefile.sas b/009_macros/mf_writefile.sas deleted file mode 100644 index f1d4d8b..0000000 --- a/009_macros/mf_writefile.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a text file using pure macro - @details Creates a text file of up to 10 lines. If further lines are - desired, feel free to [create an issue]( - https://github.com/sasjs/core/issues/new), or make a pull request! - - The use of PARMBUFF was considered for this macro, but it would have made - things problematic for writing lines containing commas. - - Usage: - - %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content) - data _null_; - infile "&sasjswork/myfile.txt"; - input; - list; - run; - - @param [in] fpath Full path to file to be created or appended to - @param [in] mode= (O) Available options are A or O as follows: - @li A APPEND mode, writes new records after the current end of the file. - @li O OUTPUT mode, writes new records from the beginning of the file. - @param [in] l1= () First line - @param [in] l2= () Second line (etc through to l10) - -

Related Macros

- @li mf_writefile.test.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ -/** @cond */ - -%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10= -)/*/STORE SOURCE*/; -%local fref rc fid i total_lines; - -/* find number of lines by reference to first non-blank param */ -%do i=10 %to 1 %by -1; - %if %str(&&l&i) ne %str() %then %goto continue; -%end; -%continue: -%let total_lines=&i; - -%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do; - %put &=fref &=fpath; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%let fid=%sysfunc(fopen(&fref,&mode)); - -%if &fid=0 %then %do; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; -%end; - -%do i=1 %to &total_lines; - %let rc=%sysfunc(fput(&fid, &&l&i)); - %let rc=%sysfunc(fwrite(&fid)); -%end; -%let rc=%sysfunc(fclose(&fid)); -%let rc=%sysfunc(filename(&fref)); - -%mend mf_writefile; -/** @endcond */ diff --git a/009_macros/mp_abort.sas b/009_macros/mp_abort.sas deleted file mode 100644 index 801e238..0000000 --- a/009_macros/mp_abort.sas +++ /dev/null @@ -1,338 +0,0 @@ -/*** HELP START ***//** - @file - @brief abort gracefully according to context - @details Configures an abort mechanism according to site specific policies or - the particulars of an environment. For instance, can stream custom - results back to the client in an STP Web App context, or completely stop - in the case of a batch run. For STP sessions - - The method used varies according to the context. Important points: - - @li should not use endsas or abort cancel in 9.4m3 WIN environments as this - can cause hung multibridge sessions and result in a frozen STP server - @li The use of endsas in 9.4m6+ windows environments for POST requests to the - STP server can result in an empty response body - @li should not use endsas in viya 3.5 as this destroys the session and cannot - fetch results (although both mv_getjoblog.sas and the @sasjs/adapter will - recognise this and fetch the log of the parent session instead) - @li STP environments must finish cleanly to avoid the log being sent to - _webout. To assist with this, we also run stpsrvset('program error', 0) - and set SYSCC=0. - Where possible, we take a unique "soft abort" approach - we open a macro - but don't close it! This works everywhere EXCEPT inside a \%include inside - a macro. For that, we recommend you use mp_include.sas to perform the - include, and then call \%mp_abort(mode=INCLUDE) from the source program (ie, - OUTSIDE of the top-parent macro). - The soft abort has become ineffective in 9.4m6 WINDOWS environments. We are - currently investigating approaches to deal with this. - - - @param [in] mac= (mp_abort.sas) To contain the name of the calling macro. Do - not use &sysmacroname as this will always resolve to MP_ABORT. - @param [out] msg= message to be returned - @param [in] iftrue= (1=1) Condition under which the macro should be executed - @param [in] errds= (work.mp_abort_errds) There is no clean way to end a - process within a %include called within a macro. Furthermore, there is no - way to test if a macro is called within a %include. To handle this - particular scenario, the %include should be switched for the mp_include.sas - macro. - This provides an indicator that we are running a macro within a \%include - (`_SYSINCLUDEFILEDEVICE`) and allows us to provide a dataset with the abort - values (msg, mac). - We can then run an abort cancel FILE to stop the include running, and pass - the dataset back to the calling program to run a regular \%mp_abort(). - The dataset will contain the following fields: - @li iftrue (1=1) - @li msg (the message) - @li mac (the mac param) - - @param [in] mode= (REGULAR) If mode=INCLUDE then the &errds dataset is checked - for an abort status. - Valid values: - @li REGULAR (default) - @li INCLUDE - - @version 9.4 - @author Allan Bowe - -

Related Macros

- @li mp_include.sas - - @cond -**//*** HELP END ***/ - -%macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1) - , errds=work.mp_abort_errds - , mode=REGULAR -)/*/STORE SOURCE*/; - -%global sysprocessmode sysprocessname sasjs_stpsrv_header_loc sasjsprocessmode; -%local fref fid i; - -%if not(%eval(%unquote(&iftrue))) %then %return; - -%put NOTE: /// mp_abort macro executing //; -%if %length(&mac)>0 %then %put NOTE- called by &mac; -%put NOTE - &msg; - -%if %symexist(_SYSINCLUDEFILEDEVICE) -/* abort cancel FILE does not restart outside the INCLUDE on Viya 3.5 */ -and %superq(SYSPROCESSNAME) ne %str(Compute Server) -%then %do; - %if "*&_SYSINCLUDEFILEDEVICE*" ne "**" %then %do; - data &errds; - iftrue='1=1'; - length mac $100 msg $5000; - mac=symget('mac'); - msg=symget('msg'); - run; - data _null_; - abort cancel FILE; - run; - %return; - %end; -%end; - -/* Web App Context */ -%if %symexist(_PROGRAM) - or %superq(SYSPROCESSNAME) = %str(Compute Server) - or &mode=INCLUDE -%then %do; - options obs=max replace mprint; - %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" - %then %do; - options nosyntaxcheck; - %end; - - %if &mode=INCLUDE %then %do; - %if %sysfunc(exist(&errds))=1 %then %do; - data _null_; - set &errds; - call symputx('iftrue',iftrue,'l'); - call symputx('mac',mac,'l'); - call symputx('msg',msg,'l'); - putlog (_all_)(=); - run; - %if (&iftrue)=0 %then %return; - %end; - %else %do; - %put &sysmacroname: No include errors found; - %return; - %end; - %end; - - /* extract log errs / warns, if exist */ - %local logloc logline; - %global logmsg; /* capture global messages */ - %if %symexist(SYSPRINTTOLOG) %then %let logloc=&SYSPRINTTOLOG; - %else %let logloc=%qsysfunc(getoption(LOG)); - proc printto log=log;run; - %let logline=0; - %if %length(&logloc)>0 %then %do; - data _null_; - infile &logloc lrecl=5000; - input; putlog _infile_; - i=1; - retain logonce 0; - if ( - _infile_=:"%str(WARN)ING" or _infile_=:"%str(ERR)OR" - ) and logonce=0 then - do; - call symputx('logline',_n_); - logonce+1; - end; - run; - /* capture log including lines BEFORE the err */ - %if &logline>0 %then %do; - data _null_; - infile &logloc lrecl=5000; - input; - i=1; - stoploop=0; - if _n_ ge &logline-15 and stoploop=0 then do until (i>22); - call symputx('logmsg',catx('\n',symget('logmsg'),_infile_)); - input; - i+1; - stoploop=1; - end; - if stoploop=1 then stop; - run; - %end; - %end; - - %if %symexist(SYS_JES_JOB_URI) %then %do; - /* setup webout for Viya */ - options nobomfile; - %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; - filename _webout temp lrecl=999999 mod; - %end; - %else %do; - filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" - name="_webout.json" lrecl=999999 mod; - %end; - %end; - %else %if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc))=0 %then %do; - options nobomfile; - /* set up http header for SASjs Server */ - %let fid=%sysfunc(fopen(&fref,A)); - %if &fid=0 %then %do; - %put %str(ERR)OR: %sysfunc(sysmsg()); - %return; - %end; - %let rc=%sysfunc(fput(&fid,%str(Content-Type: application/json))); - %let rc=%sysfunc(fwrite(&fid)); - %let rc=%sysfunc(fclose(&fid)); - %let rc=%sysfunc(filename(&fref)); - %end; - - /* send response in SASjs JSON format */ - data _null_; - file _webout mod lrecl=32000 encoding='utf-8'; - length msg syswarningtext syserrortext $32767 mode $10 ; - sasdatetime=datetime(); - msg=symget('msg'); - %if &logline>0 %then %do; - msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg')); - %end; - /* escape the escapes */ - msg=tranwrd(msg,'\','\\'); - /* escape the quotes */ - msg=tranwrd(msg,'"','\"'); - /* ditch the CRLFs as chrome complains */ - msg=compress(msg,,'kw'); - /* quote without quoting the quotes (which are escaped instead) */ - msg=cats('"',msg,'"'); - if symexist('_debug') then debug=quote(trim(symget('_debug'))); - else debug='""'; - if symget('sasjsprocessmode')='Stored Program' then mode='SASJS'; - if mode ne 'SASJS' then put '>>weboutBEGIN<<'; - put '{"SYSDATE" : "' "&SYSDATE" '"'; - put ',"SYSTIME" : "' "&SYSTIME" '"'; - put ',"sasjsAbort" : [{'; - put ' "MSG":' msg ; - put ' ,"MAC": "' "&mac" '"}]'; - put ",""SYSUSERID"" : ""&sysuserid"" "; - put ',"_DEBUG":' debug ; - if symexist('_metauser') then do; - _METAUSER=quote(trim(symget('_METAUSER'))); - put ",""_METAUSER"": " _METAUSER; - _METAPERSON=quote(trim(symget('_METAPERSON'))); - put ',"_METAPERSON": ' _METAPERSON; - end; - if symexist('SYS_JES_JOB_URI') then do; - SYS_JES_JOB_URI=quote(trim(symget('SYS_JES_JOB_URI'))); - put ',"SYS_JES_JOB_URI": ' SYS_JES_JOB_URI; - end; - _PROGRAM=quote(trim(resolve(symget('_PROGRAM')))); - put ',"_PROGRAM" : ' _PROGRAM ; - put ",""SYSCC"" : ""&syscc"" "; - syserrortext=cats(symget('syserrortext')); - if findc(syserrortext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syserrortext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syserrortext) - )))))))))))))!!'"'; - end; - else syserrortext=cats('"',syserrortext,'"'); - put ',"SYSERRORTEXT" : ' syserrortext; - put ",""SYSHOSTNAME"" : ""&syshostname"" "; - put ",""SYSJOBID"" : ""&sysjobid"" "; - put ",""SYSSCPL"" : ""&sysscpl"" "; - put ",""SYSSITE"" : ""&syssite"" "; - sysvlong=quote(trim(symget('sysvlong'))); - put ',"SYSVLONG" : ' sysvlong; - syswarningtext=cats(symget('syswarningtext')); - if findc(syswarningtext,'"\'!!'0A0D09000E0F010210111A'x) then do; - syswarningtext='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,syswarningtext) - )))))))))))))!!'"'; - end; - else syswarningtext=cats('"',syswarningtext,'"'); - put ",""SYSWARNINGTEXT"" : " syswarningtext; - put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" '; - put "}" ; - if mode ne 'SASJS' then put '>>weboutEND<<'; - run; - - %put _all_; - - %if "&sysprocessmode " = "SAS Stored Process Server " %then %do; - data _null_; - putlog 'stpsrvset program err and syscc'; - rc=stpsrvset('program error', 0); - call symputx("syscc",0,"g"); - run; - %if &sysscp=WIN - and 1=0 /* deprecating this logic until we figure out a consistent abort */ - and "%substr(%str(&sysvlong ),1,8)"="9.04.01M" - and "%substr(%str(&sysvlong ),9,1)">"5" %then %do; - /* skip approach (below) does not work in windows m6+ envs */ - endsas; - %end; - %else %do; - /** - * endsas kills 9.4m3 deployments by orphaning multibridges. - * Abort variants are ungraceful (non zero return code) - * This approach lets SAS run silently until the end :-) - * Caution - fails when called within a %include within a macro - * Use mp_include() to handle this. - */ - filename skip temp; - data _null_; - file skip; - put '%macro skip();'; - comment '%mend skip; -> fix lint '; - put '%macro skippy();'; - comment '%mend skippy; -> fix lint '; - run; - %inc skip; - %end; - %end; - %else %if "&sysprocessmode " = "SAS Compute Server " %then %do; - /* endsas kills the session making it harder to fetch results */ - data _null_; - syswarningtext=symget('syswarningtext'); - syserrortext=symget('syserrortext'); - abort_msg=symget('msg'); - syscc=symget('syscc'); - sysuserid=symget('sysuserid'); - iftrue=symget('iftrue'); - put (_all_)(/=); - call symputx('syscc',0); - abort cancel nolist; - run; - %end; - %else %do; - %abort cancel; - %end; -%end; -%else %do; - %put _all_; - %abort cancel; -%end; -%mend mp_abort; - -/** @endcond */ diff --git a/009_macros/mp_aligndecimal.sas b/009_macros/mp_aligndecimal.sas deleted file mode 100644 index 5e90f59..0000000 --- a/009_macros/mp_aligndecimal.sas +++ /dev/null @@ -1,95 +0,0 @@ -/*** HELP START ***//** - @file - @brief Apply leading blanks to align numbers vertically in a char variable - @details This is particularly useful when storing numbers (as character) that - need to be sorted. - - It works by splitting the number left and right of the decimal place, and - aligning it accordingly. A temporary variable is created as part of this - process (which is automatically dropped) - - The macro can be used only in data step, eg as follows: - - data _null_; - length myvar $50; - do i=1 to 1000 by 50; - if mod(i,2)=0 then j=ranuni(0)*i*100; - else j=i*100; - - %mp_aligndecimal(myvar,width=7) - - leading_spaces=length(myvar)-length(cats(myvar)); - putlog +leading_spaces myvar; - end; - run; - - The generated code will look something like this: - - length aligndp4e49996 $7; - if index(myvar,'.') then do; - aligndp4e49996=cats(scan(myvar,1,'.')); - aligndp4e49996=right(aligndp4e49996); - myvar=aligndp4e49996!!'.'!!cats(scan(myvar,2,'.')); - end; - else do; - aligndp4e49996=myvar; - aligndp4e49996=right(aligndp4e49996); - myvar=aligndp4e49996; - end; - drop aligndp4e49996; - - Results (myvar variable): - - 0.7683559324 - 122.8232796 - 99419.50552 - 42938.5143414 - 763.3799189 - 15170.606073 - 15083.285773 - 85443.198707 - 2022999.2251 - 12038.658867 - 1350582.6734 - 52777.258221 - 11723.347628 - 33101.268376 - 6181622.8603 - 7390614.0669 - 73384.537893 - 1788362.1016 - 2774586.2219 - 7998580.8415 - - - @param [in] var The (data step, character) variable to modify - @param [in] width= (8) The number of characters BEFORE the decimal point - -

SAS Macros

- @li mf_getuniquename.sas - -

Related Programs

- @li mp_aligndecimal.test.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_aligndecimal(var,width=8); - - %local tmpvar; - %let tmpvar=%mf_getuniquename(prefix=aligndp); - length &tmpvar $&width; - if index(&var,'.') then do; - &tmpvar=cats(scan(&var,1,'.')); - &tmpvar=right(&tmpvar); - &var=&tmpvar!!'.'!!cats(scan(&var,2,'.')); - end; - else do; - &tmpvar=cats(&var); - &tmpvar=right(&tmpvar); - &var=&tmpvar; - end; - drop &tmpvar; - -%mend mp_aligndecimal; diff --git a/009_macros/mp_appendfile.sas b/009_macros/mp_appendfile.sas deleted file mode 100644 index 4ebe478..0000000 --- a/009_macros/mp_appendfile.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief Append (concatenate) two or more files. - @details Will append one more more `appendrefs` (filerefs) to a `baseref`. - Uses a binary mechanism, so will work with any file type. For that reason - - use with care! And supply your own trailing carriage returns in each file.. - - Usage: - - filename tmp1 temp; - filename tmp2 temp; - filename tmp3 temp; - data _null_; file tmp1; put 'base file'; - data _null_; file tmp2; put 'append1'; - data _null_; file tmp3; put 'append2'; - run; - %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3) - - - @param [in] baseref= (0) Fileref of the base file (should exist) - @param [in] appendrefs= (0) One or more filerefs to be appended to the base - fileref. Space separated. - - @version 9.2 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - @li mp_binarycopy.sas - - -**//*** HELP END ***/ - -%macro mp_appendfile( - baseref=0, - appendrefs=0 -)/*/STORE SOURCE*/; - -%mp_abort(iftrue= (&baseref=0) - ,mac=&sysmacroname - ,msg=%str(Baseref NOT specified!) -) -%mp_abort(iftrue= (&appendrefs=0) - ,mac=&sysmacroname - ,msg=%str(Appendrefs NOT specified!) -) - -%local i; -%do i=1 %to %sysfunc(countw(&appendrefs)); - %mp_abort(iftrue= (&syscc>0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc) - ) - %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND) -%end; - -%mend mp_appendfile; diff --git a/009_macros/mp_applyformats.sas b/009_macros/mp_applyformats.sas deleted file mode 100644 index 0e4f883..0000000 --- a/009_macros/mp_applyformats.sas +++ /dev/null @@ -1,181 +0,0 @@ -/*** HELP START ***//** - @file - @brief Apply a set of formats to a table - @details Applies a set of formats to the metadata of one or more SAS datasets. - Can be used to migrate formats from one table to another. The input table - must contain the following columns: - - @li lib - the libref of the table to be updated - @li ds - the dataset to be updated - @li var - the variable to be updated - @li fmt - the format to apply. Missing or default ($CHAR, 8.) formats are - ignored. - - The macro will abort in the following scenarios: - - @li Libref not assigned - @li Dataset does not exist - @li Input table contains null or invalid values - - Example usage: - - data work.example; - set sashelp.prdsale; - format _all_ clear; - run; - - %mp_getcols(sashelp.prdsale,outds=work.cols) - - data work.cols2; - set work.cols; - lib='WORK'; - ds='EXAMPLE'; - var=name; - fmt=format; - keep lib ds var fmt; - run; - - %mp_applyformats(work.cols2) - - @param [in] inds The input dataset containing the formats to apply (and where - to apply them). Example structure: - |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.| - |---|---|---|---| - |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `| - |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `| - |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `| - |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `| - |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `| - |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `| - |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `| - |`WORK `|`EXAMPLE `|`QUARTER `|`8. `| - |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `| - |`WORK `|`EXAMPLE `|`YEAR `|`8. `| - - @param [out] errds= (0) Provide a libds reference here to export the - error messages to a table. In this case, they will not be printed to the - log. - -

SAS Macros

- @li mf_getengine.sas - @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mf_nobs.sas - @li mp_validatecol.sas - - -

Related Macros

- @li mp_getformats.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_applyformats(inds,errds=0 -)/*/STORE SOURCE*/; -%local outds liblist i engine lib msg ; - -/** - * Validations - */ -proc sort data=&inds; - by lib ds var fmt; -run; - -%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats); -%else %let outds=&errds; - -data &outds; - set &inds; - where fmt not in ('','.', '$', '$CHAR.','8.'); - length msg $128; - by lib ds var fmt; - if libref(lib) ne 0 then do; - msg=catx(' ','libref',lib,'is not assigned!'); - %if &errds=0 %then %do; - putlog 'ERR' +(-1) "OR: " msg; - %end; - output; - return; - end; - if exist(cats(lib,'.',ds)) ne 1 then do; - msg=catx(' ','libds',lib,'.',ds,'does not exist!'); - %if &errds=0 %then %do; - putlog 'ERR' +(-1) "OR: " msg; - %end; - output; - return; - end; - %mp_validatecol(fmt,FORMAT,is_fmt) - if is_fmt=0 then do; - msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!'); - %if &errds=0 %then %do; - putlog 'ERR' +(-1) "OR: " msg; - %end; - output; - return; - end; - - if first.ds then do; - retain dsid; - dsid=open(cats(lib,'.',ds)); - if dsid=0 then do; - msg=catx(' ','libds',lib,'.',ds,' could not be opened!'); - %if &errds=0 %then %do; - putlog 'ERR' +(-1) "OR: " msg; - %end; - output; - return; - end; - if varnum(dsid,var)<1 then do; - msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!'); - %if &errds=0 %then %do; - putlog 'ERR' +(-1) "OR: " msg; - %end; - output; - end; - end; - if last.ds then rc=close(dsid); -run; - -proc sql noprint; -select distinct lib into: liblist separated by ' ' from &inds; -%put &=liblist; -%if %length(&liblist)>0 %then %do i=1 %to %sysfunc(countw(&liblist)); - %let lib=%scan(&liblist,1); - %let engine=%mf_getengine(&lib); - %if &engine ne V9 and &engine ne BASE %then %do; - %let msg=&lib has &engine engine - formats cannot be applied; - insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ; - %if &errds=0 %then %put %str(ERR)OR: &msg; - %end; -%end; -quit; - -%if %mf_nobs(&outds)>0 %then %return; - -/** - * Validations complete - now apply the actual formats! - */ -%let fref=%mf_getuniquefileref(); -data _null_; - set &inds; - by lib ds var fmt; - where fmt not in ('','.', '$', '$CHAR.','8.'); - file &fref; - if first.lib then put 'proc datasets nolist lib=' lib ';'; - if first.ds then put ' modify ' ds ';'; - put ' format ' var fmt ';'; - if last.ds then put ' run;'; - if last.lib then put 'quit;'; -run; - -%inc &fref/source2; - -%if &errds=0 %then %do; - proc sql; - drop table &outds; -%end; - -%mend mp_applyformats; diff --git a/009_macros/mp_assert.sas b/009_macros/mp_assert.sas deleted file mode 100644 index fe7614b..0000000 --- a/009_macros/mp_assert.sas +++ /dev/null @@ -1,56 +0,0 @@ -/*** HELP START ***//** - @file - @brief Generic assertion - @details Useful in the context of writing sasjs tests. The results of the - test are _appended_ to the &outds. table. - - Example usage: - - %mp_assert(iftrue=(1=1), - desc=Obviously true - ) - - %mp_assert(iftrue=(1=0), - desc=Will fail - ) - - @param [in] iftrue= (1=1) A condition where, if true, the test is a PASS. - Else, the test is a fail. - - @param [in] desc= (Testing observations) The user provided test description - @param [out] outds= (work.test_results) The output dataset to contain the - results. If it does not exist, it will be created, with the following format: - |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| - |---|---|---| - |User Provided description|PASS|Dataset &inds contained ALL columns| - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_assert(iftrue=(1=1), - desc=0, - outds=work.test_results -)/*/STORE SOURCE*/; - - data ; - length test_description $256 test_result $4 test_comments $256; - test_description=symget('desc'); - test_comments="&sysmacroname: Test result of "!!symget('iftrue'); - %if %eval(%unquote(&iftrue)) %then %do; - test_result='PASS'; - %end; - %else %do; - test_result='FAIL'; - %end; - run; - - %local ds ; - %let ds=&syslast; - proc append base=&outds data=&ds; - run; - proc sql; - drop table &ds; - -%mend mp_assert; diff --git a/009_macros/mp_assertcols.sas b/009_macros/mp_assertcols.sas deleted file mode 100644 index 4b4ee49..0000000 --- a/009_macros/mp_assertcols.sas +++ /dev/null @@ -1,145 +0,0 @@ -/*** HELP START ***//** - @file - @brief Asserts the existence (or not) of columns - @details Useful in the context of writing sasjs tests. The results of the - test are _appended_ to the &outds. table. - - Example usage: - - %mp_assertcols(sashelp.class, - cols=name age sex, - test=ALL, - desc=check all columns exist - ) - - %mp_assertcols(sashelp.class, - cols=a b c, - test=NONE - ) - - %mp_assertcols(sashelp.class, - cols=age depth, - test=ANY - ) - -

SAS Macros

- @li mf_existds.sas - @li mf_existvarlist.sas - @li mf_getvarlist.sas - @li mf_wordsinstr1butnotstr2.sas - @li mp_abort.sas - - - @param [in] inds The input library.dataset to test for values - @param [in] cols= (0) The list of columns to check for - @param [in] desc= (0) The user provided test description - @param [in] test= (ALL) The test to apply. Valid values are: - @li ALL - Test is a PASS if ALL columns exist in &inds - @li ANY - Test is a PASS if ANY of the columns exist in &inds - @li NONE - Test is a PASS if NONE of the columns exist in &inds - @param [out] outds= (work.test_results) The output dataset to contain the - results. If it does not exist, it will be created, with the following format: - |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| - |---|---|---| - |User Provided description|PASS|Dataset &inds contained ALL columns| - - -

Related Macros

- @li mp_assertdsobs.sas - @li mp_assertcolvals.sas - @li mp_assertdsobs.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_assertcols(inds, - cols=0, - test=ALL, - desc=0, - outds=work.test_results -)/*/STORE SOURCE*/; - - %mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc - on macro entry) - ) - - %local lib ds ; - %let lib=%scan(&inds,1,%str(.)); - %let ds=%scan(&inds,2,%str(.)); - %let cols=%upcase(&cols); - - %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0) - ,mac=&sysmacroname - ,msg=%str(&lib..&ds not found!) - ) - - %mp_abort(iftrue= (&cols=0) - ,mac=&sysmacroname - ,msg=%str(No cols provided) - ) - - - %let test=%upcase(&test); - - %if &test ne ANY and &test ne ALL and &test ne NONE %then %do; - %mp_abort( - mac=&sysmacroname, - msg=%str(Invalid test - &test) - ) - %end; - - /** - * now do the actual test! - */ - %local result; - %if %mf_existVarList(&inds,&cols)=1 %then %let result=ALL; - %else %do; - %local targetcols compare; - %let targetcols=%upcase(%mf_getvarlist(&inds)); - %let compare=%mf_wordsinstr1butnotstr2( - Str1=&cols, - Str2=&targetcols - ); - %if %cmpres(&compare)=%cmpres(&cols) %then %let result=NONE; - %else %let result=SOME; - %end; - - data; - length test_description $256 test_result $4 test_comments $256; - test_description=symget('desc'); - if test_description='0' - then test_description="Testing &inds for existence of &test of: &cols"; - - test_result='FAIL'; - test_comments="&sysmacroname: &inds has &result columns "; - %if &test=ALL %then %do; - %if &result=ALL %then %do; - test_result='PASS'; - %end; - %end; - %else %if &test=ANY %then %do; - %if &result=SOME %then %do; - test_result='PASS'; - %end; - %end; - %else %if &test=NONE %then %do; - %if &result=NONE %then %do; - test_result='PASS'; - %end; - %end; - %else %do; - test_comments="&sysmacroname: Unsatisfied test condition - &test"; - %end; - run; - - %local ds; - %let ds=&syslast; - proc append base=&outds data=&ds; - run; - proc sql; - drop table &ds; - -%mend mp_assertcols; diff --git a/009_macros/mp_assertcolvals.sas b/009_macros/mp_assertcolvals.sas deleted file mode 100644 index e210a3b..0000000 --- a/009_macros/mp_assertcolvals.sas +++ /dev/null @@ -1,172 +0,0 @@ -/*** HELP START ***//** - @file - @brief Asserts the values in a column - @details Useful in the context of writing sasjs tests. The results of the - test are _appended_ to the &outds. table. - - Example usage: - - data work.checkds; - do checkval='Jane','James','Jill'; - output; - end; - run; - %mp_assertcolvals(sashelp.class.name, - checkvals=work.checkds.checkval, - desc=At least one value has a match, - test=ANYVAL - ) - - data work.check; - do val='M','F'; - output; - end; - run; - %mp_assertcolvals(sashelp.class.sex, - checkvals=work.check.val, - desc=All values have a match, - test=ALLVALS - ) - -

SAS Macros

- @li mf_existds.sas - @li mf_getuniquename.sas - @li mf_nobs.sas - @li mp_abort.sas - - - @param [in] indscol The input library.dataset.column to test for values - @param [in] checkvals= (0) A library.dataset.column value containing a UNIQUE - list of values to be compared against the source (indscol). - @param [in] desc= (Testing observations) The user provided test description - @param [in] test= (ALLVALS) The test to apply. Valid values are: - @li ALLVALS - Test is a PASS if ALL values have a match in checkvals - @li ANYVAL - Test is a PASS if at least 1 value has a match in checkvals - @li NOVAL - Test is a PASS if there are NO matches in checkvals - @param [out] outds= (work.test_results) The output dataset to contain the - results. If it does not exist, it will be created, with the following format: - |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| - |---|---|---| - |User Provided description|PASS|Column &indscol contained ALL target vals| - - -

Related Macros

- @li mp_assertdsobs.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_assertcolvals(indscol, - checkvals=0, - test=ALLVALS, - desc=mp_assertcolvals - no desc provided, - outds=work.test_results -)/*/STORE SOURCE*/; - - %mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc - on macro entry) - ) - - %local lib ds col clib cds ccol nobs; - %let lib=%scan(&indscol,1,%str(.)); - %let ds=%scan(&indscol,2,%str(.)); - %let col=%scan(&indscol,3,%str(.)); - %mp_abort(iftrue= (%mf_existds(&lib..&ds)=0) - ,mac=&sysmacroname - ,msg=%str(&lib..&ds not found!) - ) - - %mp_abort(iftrue= (&checkvals=0) - ,mac=&sysmacroname - ,msg=%str(Set CHECKVALS to a library.dataset.column containing check vals) - ) - %let clib=%scan(&checkvals,1,%str(.)); - %let cds=%scan(&checkvals,2,%str(.)); - %let ccol=%scan(&checkvals,3,%str(.)); - %mp_abort(iftrue= (%mf_existds(&clib..&cds)=0) - ,mac=&sysmacroname - ,msg=%str(&clib..&cds not found!) - ) - %let nobs=%mf_nobs(&clib..&cds); - %mp_abort(iftrue= (&nobs=0) - ,mac=&sysmacroname - ,msg=%str(&clib..&cds is empty!) - ) - - %let test=%upcase(&test); - - %if &test ne ALLVALS and &test ne ANYVAL and &test ne NOVAL %then %do; - %mp_abort( - mac=&sysmacroname, - msg=%str(Invalid test - &test) - ) - %end; - - %local result orig; - %let result=-1; - %let orig=-1; - proc sql noprint; - select count(*) into: result trimmed - from &lib..&ds - where &col not in ( - select &ccol from &clib..&cds - ); - select count(*) into: orig trimmed from &lib..&ds; - quit; - - %local notfound tmp1 tmp2; - %let tmp1=%mf_getuniquename(); - %let tmp2=%mf_getuniquename(); - - /* this is a bit convoluted - but using sql outobs=10 throws warnings */ - proc sql noprint; - create view &tmp1 as - select distinct &col - from &lib..&ds - where &col not in ( - select &ccol from &clib..&cds - ); - data &tmp2; - set &tmp1; - if _n_>10 then stop; - run; - proc sql; - select distinct &col into: notfound separated by ' ' from &tmp2; - - - %mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc after macro query) - ) - - data; - length test_description $256 test_result $4 test_comments $256; - test_description=symget('desc'); - test_result='FAIL'; - test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values " - !!"not in &clib..&cds..&ccol.. First 10 vals:"!!symget('notfound'); - %if &test=ANYVAL %then %do; - if &result < &orig then test_result='PASS'; - %end; - %else %if &test=ALLVALS %then %do; - if &result=0 then test_result='PASS'; - %end; - %else %if &test=NOVAL %then %do; - if &result=&orig then test_result='PASS'; - %end; - %else %do; - test_comments="&sysmacroname: Unsatisfied test condition - &test"; - %end; - run; - - %local ds; - %let ds=&syslast; - proc append base=&outds data=&ds; - run; - proc sql; - drop table &ds; - -%mend mp_assertcolvals; diff --git a/009_macros/mp_assertdsobs.sas b/009_macros/mp_assertdsobs.sas deleted file mode 100644 index ad49cd5..0000000 --- a/009_macros/mp_assertdsobs.sas +++ /dev/null @@ -1,121 +0,0 @@ -/*** HELP START ***//** - @file - @brief Asserts the number of observations in a dataset - @details Useful in the context of writing sasjs tests. The results of the - test are _appended_ to the &outds. table. - - Example usage: - - %mp_assertdsobs(sashelp.class) %* tests if any observations are present; - - %mp_assertdsobs(sashelp.class,test=ATLEAST 10) %* pass if >9 obs present; - - %mp_assertdsobs(sashelp.class,test=ATMOST 20) %* pass if <21 obs present; - - - @param [in] inds input dataset to test for presence of observations - @param [in] desc= (Testing observations) The user provided test description - @param [in] test= (HASOBS) The test to apply. Valid values are: - @li HASOBS - Test is a PASS if the input dataset has any observations - @li EMPTY - Test is a PASS if input dataset is empty - @li EQUALS [integer] - Test passes if row count matches the provided integer - @li ATLEAST [integer] - Test passes if row count is more than or equal to - the provided integer - @li ATMOST [integer] - Test passes if row count is less than or equal to - the provided integer - @param [out] outds= (work.test_results) The output dataset to contain the - results. If it does not exist, it will be created, with the following format: - |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| - |---|---|---| - |User Provided description|PASS|Dataset &inds has XX obs| - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_nobs.sas - @li mp_abort.sas - -

Related Macros

- @li mp_assertcolvals.sas - @li mp_assert.sas - @li mp_assertcols.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_assertdsobs(inds, - test=HASOBS, - desc=Testing observations, - outds=work.test_results -)/*/STORE SOURCE*/; - - %local nobs ds; - %let nobs=%mf_nobs(&inds); - %let test=%upcase(&test); - %let ds=%mf_getuniquename(prefix=mp_assertdsobs); - - %if %substr(&test.xxxxx,1,6)=EQUALS %then %do; - %let val=%scan(&test,2,%str( )); - %mp_abort(iftrue= (%DATATYP(&val)=CHAR) - ,mac=&sysmacroname - ,msg=%str(Invalid test - &test, expected EQUALS [integer]) - ) - %let test=EQUALS; - %end; - %else %if %substr(&test.xxxxxxx,1,7)=ATLEAST %then %do; - %let val=%scan(&test,2,%str( )); - %mp_abort(iftrue= (%DATATYP(&val)=CHAR) - ,mac=&sysmacroname - ,msg=%str(Invalid test - &test, expected ATLEAST [integer]) - ) - %let test=ATLEAST; - %end; - %else %if %substr(&test.xxxxxxx,1,7)=ATMOST %then %do; - %let val=%scan(&test,2,%str( )); - %mp_abort(iftrue= (%DATATYP(&val)=CHAR) - ,mac=&sysmacroname - ,msg=%str(Invalid test - &test, expected ATMOST [integer]) - ) - %let test=ATMOST; - %end; - %else %if &test ne HASOBS and &test ne EMPTY %then %do; - %mp_abort( - mac=&sysmacroname, - msg=%str(Invalid test - &test) - ) - %end; - - data &ds; - length test_description $256 test_result $4 test_comments $256; - test_description=symget('desc'); - test_result='FAIL'; - test_comments="&sysmacroname: Dataset &inds has &nobs observations."; - test_comments=test_comments!!" Test was "!!symget('test'); - %if &test=HASOBS %then %do; - if &nobs>0 then test_result='PASS'; - %end; - %else %if &test=EMPTY %then %do; - if &nobs=0 then test_result='PASS'; - %end; - %else %if &test=EQUALS %then %do; - if &nobs=&val then test_result='PASS'; - %end; - %else %if &test=ATLEAST %then %do; - if &nobs ge &val then test_result='PASS'; - %end; - %else %if &test=ATMOST %then %do; - if &nobs le &val then test_result='PASS'; - %end; - %else %do; - test_comments="&sysmacroname: Unsatisfied test condition - &test"; - %end; - run; - - proc append base=&outds data=&ds; - run; - - proc sql; - drop table &ds; - -%mend mp_assertdsobs; diff --git a/009_macros/mp_assertscope.sas b/009_macros/mp_assertscope.sas deleted file mode 100644 index a92fd4c..0000000 --- a/009_macros/mp_assertscope.sas +++ /dev/null @@ -1,147 +0,0 @@ -/*** HELP START ***//** - @file - @brief Used to capture scope leakage of macro variables - @details - - A common 'difficult to detect' bug in macros is where a nested macro - over-writes variables in a higher level macro. - - This assertion takes a snapshot of the macro variables before and after - a macro invocation. Differences are captured in the `&outds` table. This - makes it easy to detect whether any macro variables were modified or - changed. - - The following variables are NOT tested (as they are known, global variables - used in SASjs): - - @li &sasjs_prefix._FUNCTIONS - - Global variables are initialised in mp_init.sas - which will also trigger - "strict mode" in your SAS session. Whilst this is a default in SASjs - produced apps, if you prefer not to use this mode, simply instantiate the - following variable to prevent the macro from running: `SASJS_PREFIX` - - Example usage: - - %mp_assertscope(SNAPSHOT) - - %let oops=I did it again; - - %mp_assertscope(COMPARE, - desc=Checking macro variables against previous snapshot - ) - - This macro is designed to work alongside `sasjs test` - for more information - about this facility, visit [cli.sasjs.io/test](https://cli.sasjs.io/test). - - @param [in] action (SNAPSHOT) The action to take. Valid values: - @li SNAPSHOT - take a copy of the current macro variables - @li COMPARE - compare the current macro variables against previous values - @param [in] scope= (GLOBAL) The scope of the variables to be checked. This - corresponds to the values in the SCOPE column in `sashelp.vmacro`. - @param [in] desc= (Testing scope leakage) The user provided test description - @param [in] ignorelist= Provide a list of macro variable names to ignore from - the comparison - @param [in,out] scopeds= (work.mp_assertscope) The dataset to contain the - scope snapshot - @param [out] outds= (work.test_results) The output dataset to contain the - results. If it does not exist, it will be created, with the following format: - |TEST_DESCRIPTION:$256|TEST_RESULT:$4|TEST_COMMENTS:$256| - |---|---|---| - |User Provided description|PASS|No out of scope variables created or modified| - -

SAS Macros

- @li mf_getquotedstr.sas - @li mp_init.sas - -

Related Macros

- @li mp_assert.sas - @li mp_assertcols.sas - @li mp_assertcolvals.sas - @li mp_assertdsobs.sas - @li mp_assertscope.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_assertscope(action, - desc=Testing Scope Leakage, - scope=GLOBAL, - scopeds=work.mp_assertscope, - ignorelist=, - outds=work.test_results -)/*/STORE SOURCE*/; -%local ds test_result test_comments del add mod ilist; -%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE - SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist); - -/** - * this sets up the global vars, it will also enter STRICT mode. If this - * behaviour is not desired, simply initiate the following global macro - * variable to prevent the macro from running: SASJS_PREFIX - */ -%mp_init() - -/* get current variables */ -%if &action=SNAPSHOT %then %do; - proc sql; - create table &scopeds as - select name,offset,value - from dictionary.macros - where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist)) - order by name,offset; -%end; -%else %if &action=COMPARE %then %do; - - proc sql; - create table _data_ as - select name,offset,value - from dictionary.macros - where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist)) - order by name,offset; - - %let ds=&syslast; - - proc compare - base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist)))) - compare=&ds noprint; - run; - - %if &sysinfo=0 %then %do; - %let test_result=PASS; - %let test_comments=&scope Variables Unmodified; - %end; - %else %do; - proc sql noprint undo_policy=none; - select distinct name into: del separated by ' ' from &scopeds - where name not in (select name from &ds); - select distinct name into: add separated by ' ' from &ds - where name not in (select name from &scopeds); - select distinct a.name into: mod separated by ' ' - from &scopeds a - inner join &ds b - on a.name=b.name - and a.offset=b.offset - where a.value ne b.value; - %let test_result=FAIL; - %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del)); - %end; - - - data ; - length test_description $256 test_result $4 test_comments $256; - test_description=symget('desc'); - test_comments=symget('test_comments'); - test_result=symget('test_result'); - run; - - %let ds=&syslast; - proc append base=&outds data=&ds; - run; - proc sql; - drop table &ds; -%end; - -%mend mp_assertscope; diff --git a/009_macros/mp_base64copy.sas b/009_macros/mp_base64copy.sas deleted file mode 100644 index 1dafeeb..0000000 --- a/009_macros/mp_base64copy.sas +++ /dev/null @@ -1,117 +0,0 @@ -/*** HELP START ***//** - @file - @brief Convert a file to/from base64 format - @details Creates a new version of a file either encoded or decoded using - Base64. Inspired by this post by Michael Dixon: - https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64 - - Usage: - - filename tmp temp; - data _null_; - file tmp; - put 'base ik ally'; - run; - %mp_base64copy(inref=tmp, outref=myref, action=ENCODE) - - data _null_; - infile myref; - input; - put _infile_; - run; - - %mp_base64copy(inref=myref, outref=mynewref, action=DECODE) - - data _null_; - infile mynewref; - input; - put _infile_; - run; - - @param [in] inref= (0) Fileref of the input file (should exist) - @param [out] outref= (0) Output fileref. If it does not exist, it is created. - @param [in] action= (ENCODE) The action to take. Valid values: - @li ENCODE - Convert the file to base64 format - @li DECODE - Decode the file from base64 format - - @version 9.2 - @author Allan Bowe, source: https://github.com/sasjs/core - -

SAS Macros

- @li mp_abort.sas - - -**//*** HELP END ***/ - -%macro mp_base64copy( - inref=0, - outref=0, - action=ENCODE -)/*/STORE SOURCE*/; - -%let inref=%upcase(&inref); -%let outref=%upcase(&outref); -%let action=%upcase(&action); -%local infound outfound; -%let infound=0; -%let outfound=0; -data _null_; - set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref")); - if fileref="&inref" then call symputx('infound',1,'l'); - if fileref="&outref" then call symputx('outfound',1,'l'); -run; - -%mp_abort(iftrue= (&infound=0) - ,mac=&sysmacroname - ,msg=%str(INREF &inref NOT FOUND!) -) -%mp_abort(iftrue= (&outref=0) - ,mac=&sysmacroname - ,msg=%str(OUTREF NOT PROVIDED!) -) -%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE) - ,mac=&sysmacroname - ,msg=%str(Invalid action! Should be ENCODE OR DECODE) -) - -%if &outfound=0 %then %do; - filename &outref temp lrecl=2097088; -%end; - -%if &action=ENCODE %then %do; - data _null_; - length b64 $ 76 line $ 57; - retain line ""; - infile &inref recfm=F lrecl= 1 end=eof; - input @1 stream $char1.; - file &outref recfm=N; - substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream)); - if mod(_N_,57)=0 or EOF then do; - if eof then b64=put(trim(line),$base64X76.); - else b64=put(line, $base64X76.); - put b64 + (-1) @; - line=""; - end; - run; -%end; -%else %if &action=DECODE %then %do; - data _null_; - length filein 8 fileout 8; - filein = fopen("&inref",'I',4,'B'); - fileout = fopen("&outref",'O',3,'B'); - char= '20'x; - do while(fread(filein)=0); - length raw $4; - do i=1 to 4; - rc=fget(filein,char,1); - substr(raw,i,1)=char; - end; - rc = fput(fileout,input(raw,$base64X4.)); - rc = fwrite(fileout); - end; - rc = fclose(filein); - rc = fclose(fileout); - run; -%end; - -%mend mp_base64copy; diff --git a/009_macros/mp_binarycopy.sas b/009_macros/mp_binarycopy.sas deleted file mode 100644 index 4d35417..0000000 --- a/009_macros/mp_binarycopy.sas +++ /dev/null @@ -1,80 +0,0 @@ -/*** HELP START ***//** - @file - @brief Copy any file using binary input / output streams - @details Reads in a file byte by byte and writes it back out. Is an - os-independent method to copy files. In case of naming collision, the - default filerefs can be modified. - Note that if you have a new enough version of SAS, and you don't need features - such as APPEND, you may be better of using the fcopy() function instead. - - %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout) - - To append to a file, use the mode option, eg: - - filename tmp1 temp; - filename tmp2 temp; - data _null_; - file tmp1; - put 'stacking'; - run; - - %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND) - %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND) - - - @param [in] inloc= () quoted "path/and/filename.ext" of the file to be copied - @param [out] outloc= () quoted "path/and/filename.ext" of the file to create - @param [in] inref= (____in) If provided, this fileref takes precedence over - the `inloc` parameter - @param [out] outref= (____in) If provided, this fileref takes precedence - over the `outloc` parameter. It must already exist! - @param [in] mode= (CREATE) Valid values: - @li CREATE - Create the file (even if it already exists) - @li APPEND - Append to the file (don't overwrite) - @param [in] iftrue= (1=1) - Supply a condition for which the macro should be executed - - @returns nothing - - @version 9.2 - -**//*** HELP END ***/ - -%macro mp_binarycopy( - inloc= /* full path and filename of the object to be copied */ - ,outloc= /* full path and filename of object to be created */ - ,inref=____in /* override default to use own filerefs */ - ,outref=____out /* override default to use own filerefs */ - ,mode=CREATE - ,iftrue=%str(1=1) -)/*/STORE SOURCE*/; - %local mod; - - %if not(%eval(%unquote(&iftrue))) %then %return; - - %if &mode=APPEND %then %let mod=mod; - - /* these IN and OUT filerefs can point to anything */ - %if &inref = ____in %then %do; - filename &inref &inloc lrecl=1048576 ; - %end; - %if &outref=____out %then %do; - filename &outref &outloc lrecl=1048576 &mod; - %end; - - /* copy the file byte-for-byte */ - data _null_; - infile &inref lrecl=1 recfm=n; - file &outref &mod recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - - %if &inref = ____in %then %do; - filename &inref clear; - %end; - %if &outref=____out %then %do; - filename &outref clear; - %end; -%mend mp_binarycopy; diff --git a/009_macros/mp_chop.sas b/009_macros/mp_chop.sas deleted file mode 100644 index e51e8cf..0000000 --- a/009_macros/mp_chop.sas +++ /dev/null @@ -1,195 +0,0 @@ -/*** HELP START ***//** - @file - @brief Splits a file of ANY SIZE by reference to a search string. - @details Provide a fileref and a search string to chop off part of a file. - - Works by reading in the file byte by byte, then marking the beginning and end - of each matched string, before finally doing the chop. - - Choose whether to keep the FIRST or the LAST section of the file. Optionally, - use an OFFSET to fix the precise chop point. - - Usage: - - %let src="%sysfunc(pathname(work))/file.txt"; - %let str=Chop here!; - %let out1="%sysfunc(pathname(work))/file1.txt"; - %let out2="%sysfunc(pathname(work))/file2.txt"; - %let out3="%sysfunc(pathname(work))/file3.txt"; - %let out4="%sysfunc(pathname(work))/file4.txt"; - - data _null_; - file &src; - put "startsection&str.endsection"; - run; - - %mp_chop(&src, matchvar=str, keep=FIRST, outfile=&out1) - %mp_chop(&src, matchvar=str, keep=LAST, outfile=&out2) - %mp_chop(&src, matchvar=str, keep=FIRST, matchpoint=END, outfile=&out3) - %mp_chop(&src, matchvar=str, keep=LAST, matchpoint=END, outfile=&out4) - - filename results (&out1 &out2 &out3 &out4); - data _null_; - infile results; - input; - list; - run; - - Results: - @li `startsection` - @li `Chop here!endsection` - @li `startsectionChop here!` - @li `endsection` - - For more examples, see mp_chop.test.sas - - @param [in] infile The QUOTED path to the file on which to perform the chop - @param [in] matchvar= () Macro variable NAME containing the string to split by - @param [in] matchpoint= (START) Valid values: - @li START - chop at the beginning of the string in `matchvar`. - @li END - chop at the end of the string in `matchvar`. - @param [in] offset= (0) An adjustment to the precise chop location, by - by reference to the `matchpoint`. Should be a positive or negative integer. - @param [in] keep= (FIRST) Valid values: - @li FIRST - keep the section of the file before the chop - @li LAST - keep the section of the file after the chop - @param [in] mdebug= (0) Set to 1 to provide macro debugging - @param [out] outfile= (0) - Optional QUOTED path to the adjusted output file (avoids - overwriting the first file). - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - -

Related Macros

- @li mp_abort.sas - @li mp_gsubfile.sas - @li mp_replace.sas - @li mp_chop.test.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_chop(infile, - matchvar=, - matchpoint=START, - keep=FIRST, - offset=0, - mdebug=0, - outfile=0 -)/*/STORE SOURCE*/; - -%local fref0 dttm ds1 outref; -%let fref0=%mf_getuniquefileref(); -%let ds1=%mf_getuniquename(prefix=allchars); -%let ds2=%mf_getuniquename(prefix=startmark); - -%if &outfile=0 %then %let outfile=&infile; - -%mp_abort(iftrue= (%length(%superq(&matchvar))=0) - ,mac=mp_chop.sas - ,msg=%str(&matchvar is an empty variable) -) - -/* START */ -%let dttm=%sysfunc(datetime()); - -filename &fref0 &infile lrecl=1 recfm=n; - -/* create dataset with one char per row */ -data &ds1; - infile &fref0; - input sourcechar $char1. @@; - format sourcechar hex2.; -run; - -/* get start & stop position of first matchvar string (one row, two vars) */ -data &ds2; - /* set find string to length in bytes to cover trailing spaces */ - length string $ %length(%superq(&matchvar)); - string =symget("&matchvar"); - drop string; - - firstchar=char(string,1); - findlen=lengthm(string); /* <- for trailing bytes */ - - do _N_=1 to nobs; - set &ds1 nobs=nobs point=_N_; - if sourcechar=firstchar then do; - pos=1; - s=0; - do point=_N_ to min(_N_ + findlen -1,nobs); - set &ds1 point=point; - if sourcechar=char(string, pos) then s + 1; - else goto _leave_; - pos+1; - end; - _leave_: - if s=findlen then do; - START =_N_; - _N_ =_N_+ s - 1; - STOP =_N_; - output; - /* matched! */ - stop; - end; - end; - end; - stop; - keep START STOP; -run; - -%local split; -%let split=0; -data _null_; - set &ds2; - if "&matchpoint"='START' then do; - if "&keep"='FIRST' then mp=start; - else if "&keep"='LAST' then mp=start-1; - end; - else if "&matchpoint"='END' then do; - if "&keep"='FIRST' then mp=stop+1; - else if "&keep"='LAST' then mp=stop; - end; - split=mp+&offset; - call symputx('split',split,'l'); -%if &mdebug=1 %then %do; - put (_all_)(=); - %put &=offset; -%end; -run; -%if &split=0 %then %do; - %put &sysmacroname: No match found in &infile for string %superq(&matchvar); - %return; -%end; - -data _null_; - file &outfile recfm=n; - set &ds1; -%if &keep=FIRST %then %do; - if _n_ ge &split then stop; -%end; -%else %do; - if _n_ gt &split; -%end; - put sourcechar char1.; -run; - -%if &mdebug=0 %then %do; - filename &fref0 clear; -%end; -%else %do; - data _null_; - infile &outfile lrecl=32767; - input; - list; - if _n_>200 then stop; - run; -%end; -/* END */ -%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run; - -%mend mp_chop; diff --git a/009_macros/mp_cleancsv.sas b/009_macros/mp_cleancsv.sas deleted file mode 100644 index 9f749c9..0000000 --- a/009_macros/mp_cleancsv.sas +++ /dev/null @@ -1,79 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fixes embedded cr / lf / crlf in CSV - @details CSVs will sometimes contain lf or crlf within quotes (eg when - saved by excel). When the termstr is ALSO lf or crlf that can be tricky - to process using SAS defaults. - This macro converts any csv to follow the convention of a windows excel file, - applying CRLF line endings and converting embedded cr and crlf to lf. - - Usage: - - fileref mycsv "/path/your/csv"; - %mp_cleancsv(in=mycsv,out=/path/new.csv) - - @param [in] in= (NOTPROVIDED) - Provide path or fileref to input csv. If a period is - found, it is assumed to be a file. - @param [in] out= (NOTPROVIDED) Output path or fileref to output csv. - If a period is found, it is assumed to be a file. - @param [in] qchar= ('22'x) Quote char - hex code 22 is the double quote. - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x); -%if "&in"="NOTPROVIDED" or "&out"="NOTPROVIDED" %then %do; - %put %str(ERR)OR: Please provide valid input (&in) & output (&out) locations; - %return; -%end; - -/* presence of a period(.) indicates a physical location */ -%if %index(&in,.) %then %let in="&in"; -%if %index(&out,.) %then %let out="&out"; - -/** - * convert all cr and crlf within quotes to lf - * convert all other cr or lf to crlf - */ - data _null_; - infile &in recfm=n ; - file &out recfm=n; - retain isq iscrlf 0 qchar &qchar; - input inchar $char1. ; - if inchar=qchar then isq = mod(isq+1,2); - if isq then do; - /* inside a quote change cr and crlf to lf */ - if inchar='0D'x then do; - put '0A'x; - input inchar $char1.; - if inchar ne '0A'x then do; - put inchar $char1.; - if inchar=qchar then isq = mod(isq+1,2); - end; - end; - else put inchar $char1.; - end; - else do; - /* outside a quote, change cr and lf to crlf */ - if inchar='0D'x then do; - crblank: - put '0D0A'x; - input inchar $char1.; - if inchar='0D'x then do; - /* multiple CR indicates CR formatted file with blank lines */ - goto crblank; - end; - else if inchar ne '0A'x then do; - put inchar $char1.; - if inchar=qchar then isq = mod(isq+1,2); - end; - end; - else if inchar='0A'x then put '0D0A'x; - else put inchar $char1.; - end; - run; -%mend mp_cleancsv; -/** @endcond */ diff --git a/009_macros/mp_copyfolder.sas b/009_macros/mp_copyfolder.sas deleted file mode 100644 index b96b477..0000000 --- a/009_macros/mp_copyfolder.sas +++ /dev/null @@ -1,84 +0,0 @@ -/*** HELP START ***//** - @file - @brief A macro to recursively copy a directory - @details Performs a recursive directory listing then works from top to bottom - copying files and creating subdirectories. - - Usage: - - %let rootdir=%sysfunc(pathname(work))/demo; - %let copydir=%sysfunc(pathname(work))/demo_copy; - %mf_mkdir(&rootdir) - %mf_mkdir(&rootdir/subdir) - %mf_mkdir(&rootdir/subdir/subsubdir) - data "&rootdir/subdir/example.sas7bdat"; - run; - - %mp_copyfolder(&rootdir,©dir) - - @param [in] source Unquoted path to the folder to copy from. - @param [out] target Unquoted path to the folder to copy to. - @param [in] copymax= (MAX) Set to a positive integer to indicate the level of - subdirectory copy recursion - eg 3, to go `./3/levels/deep`. For unlimited - recursion, set to MAX. - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_isdir.sas - @li mf_mkdir.sas - @li mp_abort.sas - @li mp_dirlist.sas - -

Related Macros

- @li mp_copyfolder.test.sas - -**//*** HELP END ***/ - -%macro mp_copyfolder(source,target,copymax=MAX); - - %mp_abort(iftrue=(%mf_isdir(&source)=0) - ,mac=&sysmacroname - ,msg=%str(Source dir does not exist (&source)) - ) - - %mf_mkdir(&target) - - %mp_abort(iftrue=(%mf_isdir(&target)=0) - ,mac=&sysmacroname - ,msg=%str(Target dir could not be created (&target)) - ) - - /* prep temp table */ - %local tempds; - %let tempds=%mf_getuniquename(); - - /* recursive directory listing */ - %mp_dirlist(path=&source,outds=work.&tempds,maxdepth=©max) - - /* create folders and copy content */ - data _null_; - length msg $200; - call missing(msg); - set work.&tempds; - if _n_ = 1 then dpos+sum(length(directory),2); - filepath2="&target/"!!substr(filepath,dpos); - if file_or_folder='folder' then call execute('%mf_mkdir('!!filepath2!!')'); - else do; - length fref1 fref2 $8; - rc1=filename(fref1,filepath,'disk','recfm=n'); - rc2=filename(fref2,filepath2,'disk','recfm=n'); - if fcopy(fref1,fref2) ne 0 then do; - msg=sysmsg(); - putlog 'ERR' +(-1) "OR: Unable to copy " filepath " to " filepath2; - putlog msg=; - end; - end; - rc=filename(fref1); - rc=filename(fref2); - run; - - /* tidy up */ - proc sql; - drop table work.&tempds; - -%mend mp_copyfolder; diff --git a/009_macros/mp_coretable.sas b/009_macros/mp_coretable.sas deleted file mode 100644 index 33a5f3c..0000000 --- a/009_macros/mp_coretable.sas +++ /dev/null @@ -1,73 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create the permanent Core tables - @details Several macros in the [core](https://github.com/sasjs/core) library - make use of permanent tables. To avoid duplication in definitions, this - macro provides a central location for managing the corresponding DDL. - - Note - this macro is likely to be deprecated in future in favour of a - dedicated "datamodel" folder (prefix mddl) - - Any corresponding data would go in a seperate repo, to avoid this one - ballooning in size! - - Example usage: - - %mp_coretable(LOCKTABLE,libds=work.locktable) - - @param [in] table_ref The type of table to create. Example values: - @li DIFFTABLE - @li FILTER_DETAIL - @li FILTER_SUMMARY - @li LOCKANYTABLE - @li MAXKEYTABLE - @param [in] libds= (0) The library.dataset reference used to create the table. - If not provided, then the DDL is simply printed to the log. - -

SAS Macros

- @li mddl_dc_difftable.sas - @li mddl_dc_filterdetail.sas - @li mddl_dc_filtersummary.sas - @li mddl_dc_locktable.sas - @li mddl_dc_maxkeytable.sas - @li mf_getuniquename.sas - -

Related Macros

- @li mp_filterstore.sas - @li mp_lockanytable.sas - @li mp_retainedkey.sas - @li mp_storediffs.sas - @li mp_stackdiffs.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_coretable(table_ref,libds=0 -)/*/STORE SOURCE*/; -%local outds ; -%let outds=%sysfunc(ifc(&libds=0,%mf_getuniquename(),&libds)); -proc sql; -%if &table_ref=DIFFTABLE %then %do; - %mddl_dc_difftable(libds=&outds) -%end; -%else %if &table_ref=LOCKTABLE %then %do; - %mddl_dc_locktable(libds=&outds) -%end; -%else %if &table_ref=FILTER_SUMMARY %then %do; - %mddl_dc_filtersummary(libds=&outds) -%end; -%else %if &table_ref=FILTER_DETAIL %then %do; - %mddl_dc_filterdetail(libds=&outds) -%end; -%else %if &table_ref=MAXKEYTABLE %then %do; - %mddl_dc_maxkeytable(libds=&outds) -%end; - -%if &libds=0 %then %do; - proc sql; - describe table &syslast; - drop table &syslast; -%end; -%mend mp_coretable; diff --git a/009_macros/mp_createconstraints.sas b/009_macros/mp_createconstraints.sas deleted file mode 100644 index fcc76db..0000000 --- a/009_macros/mp_createconstraints.sas +++ /dev/null @@ -1,70 +0,0 @@ -/*** HELP START ***//** - @file mp_createconstraints.sas - @brief Creates constraints - @details Takes the output from mp_getconstraints.sas as input - - proc sql; - create table work.example( - TX_FROM float format=datetime19., - DD_TYPE char(16), - DD_SOURCE char(2048), - DD_SHORTDESC char(256), - constraint pk primary key(tx_from, dd_type,dd_source), - constraint unq unique(tx_from, dd_type), - constraint nnn not null(DD_SHORTDESC) - ); - - %mp_getconstraints(lib=work,ds=example,outds=work.constraints) - %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES) - %mp_createconstraints(inds=work.constraints,outds=created,execute=YES) - - @param [in] inds= (work.mp_getconstraints) The input table containing the - constraint info - @param [out] outds= (work.mp_createconstraints) A table containing the create - statements (create_statement column) - @param [in] execute= (NO) To actually create, use YES. - -

Related Files

- @li mp_getconstraints.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_createconstraints(inds=mp_getconstraints - ,outds=work.mp_createconstraints - ,execute=NO -)/*/STORE SOURCE*/; - -proc sort data=&inds out=&outds; - by libref table_name constraint_name; -run; - -data &outds; - set &outds; - by libref table_name constraint_name; - length create_statement $500; - if _n_=1 and "&execute"="YES" then call execute('proc sql;'); - if first.constraint_name then do; - if constraint_type='PRIMARY' then type='PRIMARY KEY'; - else type=constraint_type; - create_statement=catx(" ","alter table",libref,".",table_name - ,"add constraint",constraint_name,type,"("); - if last.constraint_name then - create_statement=cats(create_statement,column_name,");"); - else create_statement=cats(create_statement,column_name,","); - if "&execute"="YES" then call execute(create_statement); - end; - else if last.constraint_name then do; - create_statement=cats(column_name,");"); - if "&execute"="YES" then call execute(create_statement); - end; - else do; - create_statement=cats(column_name,","); - if "&execute"="YES" then call execute(create_statement); - end; - output; -run; - -%mend mp_createconstraints; diff --git a/009_macros/mp_createwebservice.sas b/009_macros/mp_createwebservice.sas deleted file mode 100644 index 423a0b1..0000000 --- a/009_macros/mp_createwebservice.sas +++ /dev/null @@ -1,32 +0,0 @@ -/*** HELP START ***//** - @file mp_createwebservice.sas - @brief Create a web service in SAS 9, Viya or SASjs Server (legacy) - @details This is actually a wrapper for mx_createwebservice.sas, remaining - for legacy purposes. For new apps, use mx_createwebservice.sas. - - -

SAS Macros

- @li mx_createwebservice.sas - - -**//*** HELP END ***/ - -%macro mp_createwebservice(path=HOME - ,name=initService - ,precode= - ,code=ft15f001 - ,desc=This service was created by the mp_createwebservice macro - ,replace=YES - ,mdebug=0 -)/*/STORE SOURCE*/; - - %mx_createwebservice(path=&path - ,name=&name - ,precode=&precode - ,code=&code - ,desc=&desc - ,replace=&replace - ,mdebug=&mdebug - ) - -%mend mp_createwebservice; diff --git a/009_macros/mp_csv2ds.sas b/009_macros/mp_csv2ds.sas deleted file mode 100644 index f98f818..0000000 --- a/009_macros/mp_csv2ds.sas +++ /dev/null @@ -1,141 +0,0 @@ -/*** HELP START ***//** - @file mp_csv2ds.sas - @brief Efficient import of arbitrary CSV using a dataset as template - @details Used to import relevant columns from a large CSV using - a dataset to provide the types and lengths. Assumes that a header - row is provided, and datarows start on line 2. Extra columns in - both the CSV and base dataset are ignored. - - Usage: - - filename mycsv temp; - data _null_; - file mycsv; - put 'name,age,nickname'; - put 'John,48,Jonny'; - put 'Jennifer,23,Jen'; - run; - - %mp_csv2ds(inref=mycsv,outds=myds,baseds=sashelp.class) - - - @param [in] inref= (0) Fileref to the CSV - @param [out] outds= (0) Output ds (lib.ds format) - @param [in] view= (NO) Set to YES or NO to determine whether the output - should be a view or not. Default is NO (not a view). - @param [in] baseds= (0) - Template dataset on which to create the input statement. - Is used to determine types, lengths, and any informats. - - @version 9.2 - @author Allan Bowe - -

SAS Macros

- @li mp_abort.sas - @li mf_existds.sas - -**//*** HELP END ***/ - -%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO); - -%mp_abort(iftrue=( &inref=0 ) - ,mac=&sysmacroname - ,msg=%str(the INREF variable must be provided) -) -%mp_abort(iftrue=( %superq(outds)=0 ) - ,mac=&sysmacroname - ,msg=%str(the OUTDS variable must be provided) -) -%mp_abort(iftrue=( &baseds=0 ) - ,mac=&sysmacroname - ,msg=%str(the BASEDS variable must be provided) -) -%mp_abort(iftrue=( %mf_existds(&baseds)=0 ) - ,mac=&sysmacroname - ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist) -) - -/* count rows */ -%local hasheader; %let hasheader=0; -data _null_; - if _N_ > 1 then do; - call symputx('hasheader',1,'l'); - stop; - end; - infile &inref; - input; -run; -%mp_abort(iftrue=( &hasheader=0 ) - ,mac=&sysmacroname - ,msg=%str(No header row in &inref) -) - -/* get the variables in the CSV */ -data _data_; - infile &inref; - input; - length name $32; - do i=1 to countc(_infile_,',')+1; - name=upcase(scan(_infile_,i,',')); - output; - end; - stop; -run; -%local csv_vars;%let csv_vars=&syslast; - -/* get the variables in the dataset */ -proc contents noprint data=&baseds - out=_data_ (keep=name type length format: informat); -run; -%local base_vars; %let base_vars=&syslast; - -proc sql undo_policy=none; -create table &csv_vars as - select a.* - ,b.type - ,b.length - ,b.format - ,b.formatd - ,b.formatl - ,b.informat - from &csv_vars a - left join &base_vars b - on a.name=upcase(b.name) - order by i; - -/* prepare the input statement */ -%local instat dropvars; -data _null_; - set &syslast end=last; - length in dropvars $32767; - retain in dropvars; - if missing(type) then do; - informat='$1.'; - dropvars=catx(' ',dropvars,name); - end; - else if missing(informat) then do; - if type=1 then informat='best.'; - else informat=cats('$',length,'.'); - end; - else informat=cats(informat,'.'); - in=catx(' ',in,name,':',informat); - if last then do; - call symputx('instat',in,'l'); - call symputx('dropvars',dropvars,'l'); - end; -run; - -/* import the CSV */ -data &outds - %if %upcase(&view)=YES %then %do; - /view=&outds - %end; - ; - infile &inref dsd firstobs=2; - input &instat; - %if %length(&dropvars)>0 %then %do; - drop &dropvars; - %end; -run; - -%mend mp_csv2ds; diff --git a/009_macros/mp_deleteconstraints.sas b/009_macros/mp_deleteconstraints.sas deleted file mode 100644 index a624d8b..0000000 --- a/009_macros/mp_deleteconstraints.sas +++ /dev/null @@ -1,54 +0,0 @@ -/*** HELP START ***//** - @file mp_deleteconstraints.sas - @brief Delete constraionts - @details Takes the output from mp_getconstraints.sas as input - - proc sql; - create table work.example( - TX_FROM float format=datetime19., - DD_TYPE char(16), - DD_SOURCE char(2048), - DD_SHORTDESC char(256), - constraint pk primary key(tx_from, dd_type,dd_source), - constraint unq unique(tx_from, dd_type), - constraint nnn not null(DD_SHORTDESC) - ); - - %mp_getconstraints(lib=work,ds=example,outds=work.constraints) - %mp_deleteconstraints(inds=work.constraints,outds=dropped,execute=YES) - - @param [in] inds= (mp_getconstraints) - The input table containing constraint info - @param [out] outds= (mp_deleteconstraints) - Table containing the drop statements (drop_statement column) - @param [in] execute= (NO) `YES|NO` - default is NO. To actually drop, use YES. - - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_deleteconstraints(inds=mp_getconstraints - ,outds=mp_deleteconstraints - ,execute=NO -)/*/STORE SOURCE*/; - -proc sort data=&inds out=&outds; - by libref table_name constraint_name; -run; - -data &outds; - set &outds; - by libref table_name constraint_name; - length drop_statement $500; - if _n_=1 and "&execute"="YES" then call execute('proc sql;'); - if first.constraint_name then do; - drop_statement=catx(" ","alter table",libref,".",table_name - ,"drop constraint",constraint_name,";"); - output; - if "&execute"="YES" then call execute(drop_statement); - end; -run; - -%mend mp_deleteconstraints; diff --git a/009_macros/mp_deletefolder.sas b/009_macros/mp_deletefolder.sas deleted file mode 100644 index b96501e..0000000 --- a/009_macros/mp_deletefolder.sas +++ /dev/null @@ -1,70 +0,0 @@ -/*** HELP START ***//** - @file - @brief A macro to delete a directory - @details Will delete all folder content (including subfolder content) and - finally, the folder itself. - - Usage: - - %let rootdir=%sysfunc(pathname(work))/demo; - %mf_mkdir(&rootdir) - %mf_mkdir(&rootdir/subdir) - %mf_mkdir(&rootdir/subdir/subsubdir) - data "&rootdir/subdir/example.sas7bdat"; - run; - - %mp_deletefolder(&rootdir) - - @param [in] folder Unquoted path to the folder to delete. - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_isdir.sas - @li mp_dirlist.sas - -

Related Macros

- @li mp_deletefolder.test.sas - -**//*** HELP END ***/ - -%macro mp_deletefolder(folder); - /* proceed if valid directory */ - %if %mf_isdir(&folder)=1 %then %do; - - /* prep temp table */ - %local tempds; - %let tempds=%mf_getuniquename(); - - /* recursive directory listing */ - %mp_dirlist(path=&folder,outds=work.&tempds, maxdepth=MAX) - - /* sort descending level so can delete folder contents before folders */ - proc sort data=work.&tempds; - by descending level; - run; - - /* ensure top level folder is removed at the end */ - proc sql; - insert into work.&tempds set filepath="&folder"; - - /* delete everything */ - data _null_; - set work.&tempds end=last; - length fref $8; - fref=''; - rc=filename(fref,filepath); - rc=fdelete(fref); - if rc then do; - msg=sysmsg(); - put "&sysmacroname:" / rc= / msg= / filepath=; - end; - rc=filename(fref); - run; - - /* tidy up */ - proc sql; - drop table work.&tempds; - - %end; - %else %put &sysmacroname: &folder: is not a valid / accessible folder. ; -%mend mp_deletefolder; diff --git a/009_macros/mp_dictionary.sas b/009_macros/mp_dictionary.sas deleted file mode 100644 index 328c284..0000000 --- a/009_macros/mp_dictionary.sas +++ /dev/null @@ -1,52 +0,0 @@ -/*** HELP START ***//** - @file mp_dictionary.sas - @brief Creates a portal (libref) into the SQL Dictionary Views - @details Provide a libref and the macro will create a series of views against - each view in the special PROC SQL dictionary libref. - - This is useful if you would like to visualise (navigate) the views in a SAS - client such as Base SAS, Enterprise Guide, or Studio (or [Data Controller]( - https://datacontroller.io)). - - It works by extracting the dictionary.dictionaries view into - YOURLIB.dictionaries, then uses that to create a YOURLIB.{viewName} for every - other dictionary.view, eg: - - proc sql; - create view YOURLIB.columns as select * from dictionary.columns; - - Usage: - - libname demo "/lib/directory"; - %mp_dictionary(lib=demo) - - Or, to just create them in WORK: - - %mp_dictionary() - - If you'd just like to browse the dictionary data model, you can also check - out [this article](https://rawsas.com/dictionary-of-dictionaries/). - - ![](https://user-images.githubusercontent.com/4420615/188278365-2987db97-0594-4a39-ac81-dbacdef5cdc8.png) - - @param [in] lib= (WORK) The libref in which to create the views - -

Related Files

- @li mp_dictionary.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_dictionary(lib=WORK)/*/STORE SOURCE*/; - %local list i mem; - proc sql noprint; - create view &lib..dictionaries as select * from dictionary.dictionaries; - select distinct memname into: list separated by ' ' from &lib..dictionaries; - %do i=1 %to %sysfunc(countw(&list,%str( ))); - %let mem=%scan(&list,&i,%str( )); - create view &lib..&mem as select * from dictionary.&mem; - %end; - quit; -%mend mp_dictionary; diff --git a/009_macros/mp_distinctfmtvalues.sas b/009_macros/mp_distinctfmtvalues.sas deleted file mode 100644 index dcc803a..0000000 --- a/009_macros/mp_distinctfmtvalues.sas +++ /dev/null @@ -1,50 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset containing distinct _formatted_ values - @details If no format is supplied, then the original value is used instead. - There is also a dependency on other macros within the Macro Core library. - Usage: - - %mp_distinctfmtvalues(libds=sashelp.class,var=age,outvar=age,outds=test) - - @param [in] libds= () input dataset - @param [in] var= (0) variable to get distinct values for - @param [out] outvar= (formatteed_value) variable to create. - @param [out] outds= (work.mp_distinctfmtvalues) dataset to create. - @param [in] varlen= (2000) length of variable to create - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_distinctfmtvalues( - libds= - ,var= - ,outvar=formatted_value - ,outds=work.mp_distinctfmtvalues - ,varlen=2000 -)/*/STORE SOURCE*/; - - %local fmt vtype; - %let fmt=%mf_getvarformat(&libds,&var); - %let vtype=%mf_getvartype(&libds,&var); - - proc sql; - create table &outds as - select distinct - %if &vtype=C & %trim(&fmt)=%str() %then %do; - &var - %end; - %else %if &vtype=C %then %do; - put(&var,&fmt) - %end; - %else %if %trim(&fmt)=%str() %then %do; - put(&var,32.) - %end; - %else %do; - put(&var,&fmt) - %end; - as &outvar length=&varlen - from &libds; -%mend mp_distinctfmtvalues; diff --git a/009_macros/mp_dropmembers.sas b/009_macros/mp_dropmembers.sas deleted file mode 100644 index 5634324..0000000 --- a/009_macros/mp_dropmembers.sas +++ /dev/null @@ -1,46 +0,0 @@ -/*** HELP START ***//** - @file - @brief Drops tables / views (if they exist) without warnings in the log - @details Useful for dropping tables when you're not sure they exist, or if - you are not sure whether they are a dataset or view. Also efficient for - dropping multiple tables / views. - - Example usage: - - proc sql; - create table data1 as select * from sashelp.class; - create view view2 as select * from sashelp.class; - %mp_dropmembers(data1 view2, libref=WORK) - - -

SAS Macros

- @li mf_isblank.sas - - - @param [in] list space separated list of datasets / views, WITHOUT libref - @param [in] libref= (WORK) Note - you can only drop from one library at a time - @param [in] iftrue= (1=1) Conditionally drop tables, eg if &debug=N - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_dropmembers( - list /* space separated list of datasets / views */ - ,libref=WORK /* can only drop from a single library at a time */ - ,iftrue=%str(1=1) -)/*/STORE SOURCE*/; - - %if not(%eval(%unquote(&iftrue))) %then %return; - - %if %mf_isblank(&list) %then %do; - %put NOTE: nothing to drop!; - %return; - %end; - - proc datasets lib=&libref nolist; - delete &list; - delete &list /mtype=view; - run; -%mend mp_dropmembers; diff --git a/009_macros/mp_ds2cards.sas b/009_macros/mp_ds2cards.sas deleted file mode 100644 index 705e031..0000000 --- a/009_macros/mp_ds2cards.sas +++ /dev/null @@ -1,291 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a CARDS file from a SAS dataset. - @details Uses dataset attributes to convert all data into datalines. - Running the generated file will rebuild the original dataset. Includes - support for large decimals, binary data, PROCESSED_DTTM columns, and - alternative encoding. If the input dataset is empty, the cards file will - still be created. - - Additional support to generate a random sample and max rows. - - Usage: - - %mp_ds2cards(sashelp.class - , tgt_ds=work.class - , cards_file= "C:\temp\class.sas" - , showlog=NO - , maxobs=5 - ) - - TODO: - - labelling the dataset - - explicity setting a unix LF - - constraints / indexes etc - - @param [in] base_ds Should be two level - eg work.blah. This is the table - that is converted to a cards file. - @param [in] tgt_ds= Table that the generated cards file would create. - Optional - if omitted, will be same as BASE_DS. - @param [out] cards_file= ("%sysfunc(pathname(work))/cardgen.sas") Location in - which to write the (.sas) cards file - @param [in] maxobs= (max) To limit output to the first maxobs - observations, enter an integer here. - @param [in] random_sample= (NO) Set to YES to generate a random sample of - data. Can be quite slow. - @param [in] showlog= (YES) Whether to show generated cards file in the SAS - log. Valid values: - @li YES - @li NO - @param [in] outencoding= Provide encoding value for file statement (eg utf-8) - @param [in] append= (NO) If NO then will rebuild the cards file if it already - exists, otherwise will append to it. Used by the mp_lib2cards.sas macro. - -

Related Macros

- @li mp_lib2cards.sas - @li mp_ds2inserts.sas - @li mp_mdtablewrite.sas - - @version 9.2 - @author Allan Bowe - @cond -**//*** HELP END ***/ - -%macro mp_ds2cards(base_ds, tgt_ds= - ,cards_file="%sysfunc(pathname(work))/cardgen.sas" - ,maxobs=max - ,random_sample=NO - ,showlog=YES - ,outencoding= - ,append=NO -)/*/STORE SOURCE*/; -%local i setds nvars; - -%if not %sysfunc(exist(&base_ds)) %then %do; - %put %str(WARN)ING: &base_ds does not exist; - %return; -%end; - -%if %index(&base_ds,.)=0 %then %let base_ds=WORK.&base_ds; -%if (&tgt_ds = ) %then %let tgt_ds=&base_ds; -%if %index(&tgt_ds,.)=0 %then %let tgt_ds=WORK.%scan(&base_ds,2,.); -%if ("&outencoding" ne "") %then %let outencoding=encoding="&outencoding"; -%if ("&append" = "" or "&append" = "NO") %then %let append=; -%else %let append=mod; - -/* get varcount */ -%let nvars=0; -proc sql noprint; -select count(*) into: nvars from dictionary.columns - where upcase(libname)="%scan(%upcase(&base_ds),1)" - and upcase(memname)="%scan(%upcase(&base_ds),2)"; -%if &nvars=0 %then %do; - %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.; - %return; -%end; - -/* get indexes */ -proc sort - data=sashelp.vindex( - where=(upcase(libname)="%scan(%upcase(&base_ds),1)" - and upcase(memname)="%scan(%upcase(&base_ds),2)") - ) - out=_data_; - by indxname indxpos; -run; - -%local indexes; -data _null_; - set &syslast end=last; - if _n_=1 then call symputx('indexes','(index=(','l'); - by indxname indxpos; - length vars $32767 nom uni $8; - retain vars; - if first.indxname then do; - idxcnt+1; - nom=''; - uni=''; - vars=name; - end; - else vars=catx(' ',vars,name); - if last.indxname then do; - if nomiss='yes' then nom='/nomiss'; - if unique='yes' then uni='/unique'; - call symputx('indexes' - ,catx(' ',symget('indexes'),indxname,'=(',vars,')',nom,uni) - ,'l'); - end; - if last then call symputx('indexes',cats(symget('indexes'),'))'),'l'); -run; - - -data;run; -%let setds=&syslast; -proc sql -%if %datatyp(&maxobs)=NUMERIC %then %do; - outobs=&maxobs; -%end; - ; - create table &setds as select * from &base_ds -%if &random_sample=YES %then %do; - order by ranuni(42) -%end; - ; -reset outobs=max; -create table datalines1 as - select name,type,length,varnum,format,label from dictionary.columns - where upcase(libname)="%upcase(%scan(&base_ds,1))" - and upcase(memname)="%upcase(%scan(&base_ds,2))"; - -/** - Due to long decimals cannot use best. format - So - use bestd. format and then use character functions to strip trailing - zeros, if NOT an integer or missing!! Cannot use int() as it upsets - note2err when there are missings. - resolved code = ifc( mod(coalesce(VARIABLE,0),1)=0 - ,put(VARIABLE,best32.) - ,substrn(put(VARIABLE,bestd32.),1 - ,findc(put(VARIABLE,bestd32.),'0','TBK'))); -**/ - -data datalines_2; - format dataline $32000.; - set datalines1 (where=(upcase(name) not in - ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM'))); - if type='num' then dataline= - cats('ifc(mod(coalesce(',name,',0),1)=0 - ,put(',name,',best32.-l) - ,substrn(put(',name,',bestd32.-l),1 - ,findc(put(',name,',bestd32.-l),"0","TBK")))'); - /** - * binary data must be converted, to store in text format. It is identified - * by the presence of the $HEX keyword in the format. - */ - else if upcase(format)=:'$HEX' then - dataline=cats('put(trim(',name,'),',format,')'); - /** - * There is no easy way to store line breaks in a cards file. - * To discuss this, use: https://github.com/sasjs/core/issues/80 - * Removing all nonprintables with kw (keep writeable) - */ - else dataline=cats('compress(',name,', ,"kw")'); -run; - -proc sql noprint; -select dataline into: datalines separated by ',' from datalines_2; - -%local - process_dttm_flg - valid_from_dttm_flg - valid_to_dttm_flg -; -%let process_dttm_flg = N; -%let valid_from_dttm_flg = N; -%let valid_to_dttm_flg = N; -data _null_; - set datalines1 ; -/* build attrib statement */ - if type='char' then type2='$'; - if strip(format) ne '' then format2=cats('format=',format); - if strip(label) ne '' then label2=cats('label=',quote(trim(label))); - str1=catx(' ',(put(name,$33.)||'length=') - ,put(cats(type2,length),$7.)||format2,label2); - - -/* Build input statement */ - if upcase(format)=:'$HEX' then type3=':'!!format; - else if type='char' then type3=':$char.'; - str2=put(name,$33.)||type3; - - - if(upcase(name) = "PROCESSED_DTTM") then - call symputx("process_dttm_flg", "Y", "L"); - if(upcase(name) = "VALID_FROM_DTTM") then - call symputx("valid_from_dttm_flg", "Y", "L"); - if(upcase(name) = "VALID_TO_DTTM") then - call symputx("valid_to_dttm_flg", "Y", "L"); - - - call symputx(cats("attrib_stmt_", put(_N_, 8.)), str1, "L"); - call symputx(cats("input_stmt_", put(_N_, 8.)) - , ifc(upcase(name) not in - ('PROCESSED_DTTM','VALID_FROM_DTTM','VALID_TO_DTTM'), str2, ""), "L"); -run; - -data _null_; - file &cards_file. &outencoding lrecl=32767 termstr=nl &append; - length __attrib $32767; - if _n_=1 then do; - put '/**'; - put ' @file'; - put " @brief Datalines for %upcase(%scan(&base_ds,2)) dataset"; - put " @details Generated by %nrstr(%%)mp_ds2cards()"; - put " Source: https://github.com/sasjs/core"; - put ' @cond '; - put '**/'; - put "data &tgt_ds &indexes;"; - put "attrib "; - %do i = 1 %to &nvars; - __attrib=symget("attrib_stmt_&i"); - put __attrib; - %end; - put ";"; - - %if &process_dttm_flg. eq Y %then %do; - put 'retain PROCESSED_DTTM %sysfunc(datetime());'; - %end; - %if &valid_from_dttm_flg. eq Y %then %do; - put 'retain VALID_FROM_DTTM &low_date;'; - %end; - %if &valid_to_dttm_flg. eq Y %then %do; - put 'retain VALID_TO_DTTM &high_date;'; - %end; - if __nobs=0 then do; - put 'call missing(of _all_);/* avoid uninitialised notes */'; - put 'stop;'; - put 'run;'; - end; - else do; - put "infile cards dsd;"; - put "input "; - %do i = 1 %to &nvars.; - %if(%length(&&input_stmt_&i..)) %then - put " &&input_stmt_&i.."; - ; - %end; - put ";"; - put 'missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;'; - put "datalines4;"; - end; - end; - set &setds end=__lastobs nobs=__nobs; -/* remove all formats for write purposes - some have long underlying decimals */ - format _numeric_ best30.29; - length __dataline $32767; - __dataline=catq('cqtmb',&datalines); - put __dataline; - if __lastobs then do; - put ';;;;'; - put 'run;'; - put '/** @endcond **/'; - stop; - end; -run; -proc sql; - drop table &setds; -quit; - -%if &showlog=YES %then %do; - data _null_; - infile &cards_file lrecl=32767; - input; - put _infile_; - run; -%end; - -%put NOTE: CARDS FILE SAVED IN:; -%put NOTE-;%put NOTE-; -%put NOTE- %sysfunc(dequote(&cards_file.)); -%put NOTE-;%put NOTE-; -%mend mp_ds2cards; -/** @endcond **/ diff --git a/009_macros/mp_ds2ddl.sas b/009_macros/mp_ds2ddl.sas deleted file mode 100644 index 02aa6b7..0000000 --- a/009_macros/mp_ds2ddl.sas +++ /dev/null @@ -1,42 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fetches DDL for a specific table - @details Uses mp_getddl under the hood - - @param [in] libds library.dataset to create ddl for - @param [in] fref= (getddl) the fileref to which to _append_ the DDL. If it - does not exist, it will be created. - @param [in] flavour= (SAS) The type of DDL to create. Options: - @li SAS - @li TSQL - - @param [in]showlog= (NO) Set to YES to show the DDL in the log - @param [in] schema= () Choose a preferred schema name (default is to use - actual schema, else libref) - @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created with - native datetime2 format or regular decimal type - -

SAS Macros

- @li mp_getddl.sas - -**//*** HELP END ***/ - -%macro mp_ds2ddl(libds,fref=getddl,flavour=SAS,showlog=YES,schema= - ,applydttm=NO -)/*/STORE SOURCE*/; - -%local libref; -%let libds=%upcase(&libds); -%let libref=%scan(&libds,1,.); -%if &libref=&libds %then %let libds=WORK.&libds; - -%mp_getddl(%scan(&libds,1,.) - ,%scan(&libds,2,.) - ,fref=&fref - ,flavour=SAS - ,showlog=&showlog - ,schema=&schema - ,applydttm=&applydttm -) - -%mend mp_ds2ddl; diff --git a/009_macros/mp_ds2fmtds.sas b/009_macros/mp_ds2fmtds.sas deleted file mode 100644 index c553f3f..0000000 --- a/009_macros/mp_ds2fmtds.sas +++ /dev/null @@ -1,105 +0,0 @@ -/*** HELP START ***//** - @file - @brief Converts every value in a dataset to formatted value - @details Converts every value to it's formatted value. All variables will - become character, and will be in the same order as the original dataset. - - Lengths will be adjusted according to the format lengths, where applicable. - - Usage: - - %mp_ds2fmtds(sashelp.cars,work.cars) - %mp_ds2fmtds(sashelp.vallopt,vw_vallopt) - - @param [in] libds The library.dataset to be converted - @param [out] outds The dataset to create. - -

SAS Macros

- @li mf_existds.sas - -

Related Macros

- @li mp_jsonout.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_ds2fmtds(libds, outds -)/*/STORE SOURCE*/; - -/* validations */ - -%if not %mf_existds(libds=&libds) %then %do; - %put %str(WARN)ING: &libds does not exist as either a VIEW or DATASET; - %return; -%end; -%if %index(&libds,.)=0 %then %let libds=WORK.&libds; - -/* grab metadata */ -proc contents noprint data=&libds - out=_data_(keep=name type length format formatl formatd varnum); -run; -proc sort; - by varnum; -run; - -/* prepare formats and varnames */ -data _null_; - set &syslast end=last; - name=upcase(name); - /* fix formats */ - if type=2 or type=6 then do; - length fmt $49.; - if format='' then fmt=cats('$',length,'.'); - else if formatl=0 then fmt=cats(format,'.'); - else fmt=cats(format,formatl,'.'); - newlen=max(formatl,length); - end; - else do; - if format='' then fmt='best.'; - else if formatl=0 then fmt=cats(format,'.'); - else if formatd=0 then fmt=cats(format,formatl,'.'); - else fmt=cats(format,formatl,'.',formatd); - /* needs to be wide, for datetimes etc */ - newlen=max(length,formatl,24); - end; - /* 32 char unique name */ - newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27); - - call symputx(cats('name',_n_),name,'l'); - call symputx(cats('newname',_n_),newname,'l'); - call symputx(cats('len',_n_),newlen,'l'); - call symputx(cats('fmt',_n_),fmt,'l'); - call symputx(cats('type',_n_),type,'l'); - if last then call symputx('nobs',_n_,'l'); -run; - -/* clean up */ -proc sql; -drop table &syslast; - -%if &nobs=0 %then %do; - %put Dataset &libds has no columns! - data &outds; - set &libds; - run; - %return; -%end; - -data &outds; - /* rename on entry */ - set &libds(rename=( -%local i; -%do i=1 %to &nobs; - &&name&i=&&newname&i -%end; - )); -%do i=1 %to &nobs; - length &&name&i $&&len&i; - &&name&i=left(put(&&newname&i,&&fmt&i)); - drop &&newname&i; -%end; - if _error_ then call symputx('syscc',1012); -run; - -%mend mp_ds2fmtds; diff --git a/009_macros/mp_ds2inserts.sas b/009_macros/mp_ds2inserts.sas deleted file mode 100644 index f24a621..0000000 --- a/009_macros/mp_ds2inserts.sas +++ /dev/null @@ -1,180 +0,0 @@ -/*** HELP START ***//** - @file - @brief Export a dataset to SQL insert statements - @details Converts dataset values to SQL insert statements for use across - multiple database types. - - Usage: - - %mp_ds2inserts(sashelp.class,outref=myref,outds=class) - data class; - set sashelp.class; - stop; - proc sql; - %inc myref; - - @param [in] ds The dataset to be exported - @param [in] maxobs= (max) The max number of inserts to create - @param [out] outref= (0) The output fileref. If it does not exist, it is - created. If it does exist, new records are APPENDED. - @param [out] schema= (0) The library (or schema) in which the target table is - located. If not provided, is ignored. - @param [out] outds= (0) The output table to load. If not provided, will - default to the table in the &ds parameter. - @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid - options: - @li SAS (default) - suitable for regular proc sql - @li PGSQL - Used for Postgres databases - @param [in] applydttm= (YES) If YES, any columns using datetime formats will - be converted to native DB datetime literals - -

SAS Macros

- @li mf_existfileref.sas - @li mf_getvarcount.sas - @li mf_getvarformat.sas - @li mf_getvarlist.sas - @li mf_getvartype.sas - - @version 9.2 - @author Allan Bowe (credit mjsq) -**//*** HELP END ***/ - -%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max - ,applydttm=YES -)/*/STORE SOURCE*/; - -%if not %sysfunc(exist(&ds)) %then %do; - %put %str(WAR)NING: &ds does not exist; - %return; -%end; - -%if not %sysfunc(exist(&ds)) %then %do; - %put %str(WAR)NING: &ds does not exist; - %return; -%end; - -%if %index(&ds,.)=0 %then %let ds=WORK.&ds; - -%let flavour=%upcase(&flavour); -%if &flavour ne SAS and &flavour ne PGSQL %then %do; - %put %str(WAR)NING: &flavour is not supported; - %return; -%end; - -%if &outref=0 %then %do; - %put %str(WAR)NING: Please provide a fileref; - %return; -%end; -%if %mf_existfileref(&outref)=0 %then %do; - filename &outref temp lrecl=66000; -%end; - -%if &schema=0 %then %let schema=; -%else %let schema=&schema..; - -%if &outds=0 %then %let outds=%scan(&ds,2,.); - -%local nobs; -proc sql noprint; -select count(*) into: nobs TRIMMED from &ds; -%if &nobs=0 %then %do; - data _null_; - file &outref mod; - put "/* No rows found in &ds */"; - run; -%end; - -%local vars; -%let vars=%mf_getvarcount(&ds); -%if &vars=0 %then %do; - data _null_; - file &outref mod; - put "/* No columns found in &schema.&ds */"; - run; - %return; -%end; -%else %if &vars>1600 and &flavour=PGSQL %then %do; - data _null_; - file &fref mod; - put "/* &schema.&ds contains &vars vars */"; - put "/* Postgres cannot handle tables with over 1600 vars */"; - put "/* No inserts will be generated for this table */"; - run; - %return; -%end; - -%local varlist varlistcomma; -%let varlist=%mf_getvarlist(&ds); -%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double); - -/* next, export data */ -data _null_; - file &outref mod ; - if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */"; - set &ds; - %if &maxobs ne max %then %do; - if _n_>&maxobs then stop; - %end; - length _____str $32767; - call missing(_____str); - format _numeric_ best.; - format _character_ ; - %local i comma var vtype vfmt; - %do i=1 %to %sysfunc(countw(&varlist)); - %let var=%scan(&varlist,&i); - %let vtype=%mf_getvartype(&ds,&var); - %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1)); - %if &i=1 %then %do; - %if &flavour=SAS %then %do; - put "insert into &schema.&outds set "; - put " &var="@; - %end; - %else %if &flavour=PGSQL %then %do; - _____str=cats( - "INSERT INTO &schema.&outds (" - ,symget('varlistcomma') - ,") VALUES (" - ); - put _____str; - put " "@; - %end; - %end; - %else %do; - %if &flavour=SAS %then %do; - put " ,&var="@; - %end; - %else %if &flavour=PGSQL %then %do; - put " ,"@; - %end; - %end; - %if &vtype=N %then %do; - %if &flavour=SAS %then %do; - put &var; - %end; - %else %if &flavour=PGSQL %then %do; - if missing(&var) then put 'NULL'; - %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME" - %then %do; - else put "TIMESTAMP '" &var E8601DT25.6 "'"; - %end; - %else %do; - else put &var; - %end; - %end; - %end; - %else %do; - _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'"; - put _____str; - %end; - %end; - %if &flavour=SAS %then %do; - put ';'; - %end; - %else %if &flavour=PGSQL %then %do; - put ');'; - %end; - - if _n_=&nobs then put /; -run; - -%mend mp_ds2inserts; diff --git a/009_macros/mp_ds2md.sas b/009_macros/mp_ds2md.sas deleted file mode 100644 index ca58489..0000000 --- a/009_macros/mp_ds2md.sas +++ /dev/null @@ -1,104 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a Markdown Table from a dataset - @details A markdown table is a simple table representation for use in - documents written in markdown format. - - An online generator is available here: - https://www.tablesgenerator.com/markdown_tables - - This structure is also used by the Macro Core library for documenting input/ - output datasets, as well as the sasjs/cli tool for documenting inputs/outputs - for web services. - - We take the standard definition one step further by embedding the informat - in the table header row, like so: - - |var1:$32|var2:best.|var3:date9.| - |---|---|---| - |some text|42|01JAN1960| - |blah|1|31DEC1999| - - Which resolves to: - - |var1:$32|var2:best.|var3:date9.| - |---|---|---| - |some text|42|01JAN1960| - |blah|1|31DEC1999| - - - Usage: - - %mp_ds2md(sashelp.class) - - @param [in] libds the library / dataset to create or read from. - @param [out] outref= (mdtable) Fileref to contain the markdown - @param [out] showlog= (YES) Set to NO to avoid printing markdown to the log - -

SAS Macros

- @li mf_getvarlist.sas - @li mf_getvarformat.sas - - @version 9.3 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_ds2md( - libds, - outref=mdtable, - showlog=YES -)/*/STORE SOURCE*/; - -/* check fileref is assigned */ -%if %sysfunc(fileref(&outref)) > 0 %then %do; - filename &outref temp; -%end; - -%local vars; -%let vars=%upcase(%mf_getvarlist(&libds)); - -%if %trim(X&vars)=X %then %do; - %put &sysmacroname: Table &libds has no columns!!; - %return; -%end; - -/* create the header row */ -data _null_; - file &outref; - length line $32767; - call missing(line); - put '|' -%local i var fmt; -%do i=1 %to %sysfunc(countw(&vars)); - %let var=%scan(&vars,&i); - %let fmt=%lowcase(%mf_getvarformat(&libds,&var,force=1)); - "&var:&fmt|" -%end; - ; - put '|' -%do i=1 %to %sysfunc(countw(&vars)); - "---|" -%end; - ; -run; - -/* write out the data */ -data _null_; - file &outref mod dlm='|' lrecl=32767; - set &libds ; - length line $32767; - line='|`'!!cats(%mf_getvarlist(&libds,dlm=%str(%)!!' `|`'!!cats%()))!!' `|'; - put line; -run; - -%if %upcase(&showlog)=YES %then %do; - options ps=max lrecl=max; - data _null_; - infile &outref; - if _n_=1 then putlog "# &libds" /; - input; - putlog _infile_; - run; -%end; - -%mend mp_ds2md; diff --git a/009_macros/mp_ds2squeeze.sas b/009_macros/mp_ds2squeeze.sas deleted file mode 100644 index 407a016..0000000 --- a/009_macros/mp_ds2squeeze.sas +++ /dev/null @@ -1,120 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create a smaller version of a dataset, without data loss - @details This macro will scan the input dataset and create a new one, that - has the minimum variable lengths needed to store the data without data loss. - - Inspiration was taken from [How to Reduce the Disk Space Required by a - SAS® Data Set](https://www.lexjansen.com/nesug/nesug06/io/io18.pdf) by - Selvaratnam Sridharma. The end of the referenced paper presents a macro named - "squeeze", hence the nomenclature. - - Usage: - - data big; - length my big $32000; - do i=1 to 1e4; - my=repeat('oh my',100); - big='dawg'; - special=._; - output; - end; - run; - - %mp_ds2squeeze(work.big,outds=work.smaller) - - The following will also be printed to the log (exact values may differ - depending on your OS and COMPRESS settings): - - > MP_DS2SQUEEZE: work.big was 625MB - - > MP_DS2SQUEEZE: work.smaller is 5MB - - @param [in] libds The library.dataset to be squeezed - @param [out] outds= (work.mp_ds2squeeze) The squeezed dataset to create - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

SAS Macros

- @li mf_getfilesize.sas - @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - @li mp_getmaxvarlengths.sas - -

Related Programs

- @li mp_ds2squeeze.test.sas - - @version 9.3 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_ds2squeeze( - libds, - outds=work.mp_ds2squeeze, - mdebug=0 -)/*/STORE SOURCE*/; -%local dbg source; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %do; - %let dbg=*; - %let source=/source2; -%end; - -%local optval ds fref startsize; -%let ds=%mf_getuniquename(); -%let fref=%mf_getuniquefileref(); -%let startsize=%mf_getfilesize(libds=&libds,format=yes); - -%mp_getmaxvarlengths(&libds,outds=&ds) - -data _null_; - set &ds end=last; - file &fref; - /* grab the types */ - retain dsid; - if _n_=1 then dsid=open("&libds",'is'); - if dsid le 0 then do; - msg=sysmsg(); - put msg=; - stop; - end; - type=vartype(dsid,varnum(dsid, name)); - if last then rc=close(dsid); - /* write out the length statement */ - if _n_=1 then put 'length '; - length len $6; - if type='C' then do; - if maxlen=0 then len='$1'; - else len=cats('$',maxlen); - end; - else do; - if maxlen=0 then len='3'; - else len=cats(maxlen); - end; - put ' ' name ' ' len; - if last then put ';'; -run; - -/* configure varlenchk - as we are explicitly shortening the variables */ -%let optval=%sysfunc(getoption(varlenchk)); -options varlenchk=NOWARN; - -data &outds; - %inc &fref &source; - set &libds; -run; - -options varlenchk=&optval; - -%if &mdebug=0 %then %do; - proc sql; - drop table &ds; - filename &fref clear; -%end; - -%put &sysmacroname: &libds was &startsize; -%put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes); - -%mend mp_ds2squeeze; diff --git a/009_macros/mp_dsmeta.sas b/009_macros/mp_dsmeta.sas deleted file mode 100644 index 9bed16c..0000000 --- a/009_macros/mp_dsmeta.sas +++ /dev/null @@ -1,110 +0,0 @@ -/*** HELP START ***//** - @file - @brief Export dataset metadata to a single output table - @details Exports the dataset attributes and enginehost information, then - converts the datasets into a single output table in the following format: - -|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.| -|---|---|---| -|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `| -|`ATTRIBUTES `|`Observations `|`19 `| -|`ATTRIBUTES `|`Member Type `|`DATA `| -|`ATTRIBUTES `|`Variables `|`5 `| -|`ATTRIBUTES `|`Engine `|`V9 `| -|`ATTRIBUTES `|`Indexes `|`0 `| -|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `| -|`ATTRIBUTES `|`Observation Length `|`40 `| -|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `| -|`ATTRIBUTES `|`Deleted Observations `|`0 `| -|`ATTRIBUTES `|`Protection `|`. `| -|`ATTRIBUTES `|`Compressed `|`NO `| -|`ATTRIBUTES `|`Data Set Type `|`. `| -|`ATTRIBUTES `|`Sorted `|`NO `| -|`ATTRIBUTES `|`Label `|`Student Data `| -|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `| -|`ATTRIBUTES `|`Encoding `|`us-ascii ASCII (ANSI) `| -|`ENGINEHOST `|`Data Set Page Size `|`65536 `| -|`ENGINEHOST `|`Number of Data Set Pages `|`1 `| -|`ENGINEHOST `|`First Data Page `|`1 `| -|`ENGINEHOST `|`Max Obs per Page `|`1632 `| -|`ENGINEHOST `|`Obs in First Data Page `|`19 `| -|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `| -|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `| -|`ENGINEHOST `|`Release Created `|`9.0401M7 `| -|`ENGINEHOST `|`Host Created `|`Linux `| -|`ENGINEHOST `|`Inode Number `|`28314616 `| -|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `| -|`ENGINEHOST `|`Owner Name `|`sas `| -|`ENGINEHOST `|`File Size `|`128KB `| -|`ENGINEHOST `|`File Size (bytes) `|`131072 `| - - Example usage: - - %mp_dsmeta(sashelp.class,outds=work.mymeta) - proc print data=work.mymeta; - run; - - For more details on creating datasets from PROC CONTENTS check out this - excellent [paper]( - https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by - [Louise Hadden](https://www.linkedin.com/in/louisehadden/). - - @param [in] libds The library.dataset to export the metadata for - @param [out] outds= (work.dsmeta) The output table to contain the metadata - -

Related Files

- @li mp_dsmeta.test.sas - @li mp_getcols.sas - @li mp_getdbml.sas - @li mp_getddl.sas - @li mp_getformats.sas - @li mp_getpk.sas - @li mp_guesspk.sas - -**//*** HELP END ***/ - -%macro mp_dsmeta(libds,outds=work.dsmeta); - -%local ds1 ds2; -data;run; %let ds1=&syslast; -data;run; %let ds2=&syslast; - -/* setup the ODS capture */ -ods output attributes=&ds1 enginehost=&ds2; - -/* export the metadata */ -proc contents data=&libds; -run; - -/* load it into a single table */ -data &outds (keep=ods_table name value); - length ods_table $10 name label2 label1 label $100 - value cvalue cvalue1 cvalue2 $1000 - nvalue nvalue1 nvalue2 8; - if _n_=1 then call missing (of _all_); - * putlog (_all_)(=); - set &ds1 (in=atrs) &ds2 (in=eng); - if atrs then do; - ods_table='ATTRIBUTES'; - name=coalescec(label1,label); - value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); - output; - if label2 ne '' then do; - name=label2; - value=coalescec(cvalue2,put(nvalue2,best.)); - output; - end; - end; - else if eng then do; - ods_table='ENGINEHOST'; - name=coalescec(label1,label); - value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); - output; - end; -run; - -proc sql; -drop table &ds1, &ds2; - -%mend mp_dsmeta; - diff --git a/009_macros/mp_filtergenerate.sas b/009_macros/mp_filtergenerate.sas deleted file mode 100644 index 659d6cc..0000000 --- a/009_macros/mp_filtergenerate.sas +++ /dev/null @@ -1,105 +0,0 @@ -/*** HELP START ***//** - @file - @brief Generates a filter clause from an input table, to a fileref - @details Uses the input table to generate an output filter clause. - This feature is used to create dynamic dropdowns in [Data Controller for SAS®]( - https://datacontroller.io). The input table should be in the format below: - - |GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$4000| - |---|---|---|---|---|---| - |AND|AND|1|AGE|=|12| - |AND|AND|1|SEX|<=|'M'| - |AND|OR|2|Name|NOT IN|('Jane','Alfred')| - |AND|OR|2|Weight|>=|7| - - Note - if the above table is received from an external client, the values - should first be validated using the mp_filtercheck.sas macro to avoid risk - of SQL injection. - - To generate the filter, run the following code: - - data work.filtertable; - infile datalines4 dsd; - input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$4000.; - datalines4; - AND,AND,1,AGE,=,12 - AND,AND,1,SEX,<=,"'M'" - AND,OR,2,Name,NOT IN,"('Jane','Alfred')" - AND,OR,2,Weight,>=,7 - ;;;; - run; - - %mp_filtergenerate(work.filtertable,outref=myfilter) - - data _null_; - infile myfilter; - input; - put _infile_; - run; - - Will write the following query to the log: - - > ( - > AGE = 12 - > AND - > SEX <= 'M' - > ) AND ( - > Name NOT IN ('Jane','Alfred') - > OR - > Weight >= 7 - > ) - - @param [in] inds The input table with query values - @param [out] outref= (filter) The output fileref to contain the filter clause. - Will be created (or replaced). - -

Related Macros

- @li mp_filtercheck.sas - @li mp_filtervalidate.sas - -

SAS Macros

- @li mp_abort.sas - @li mf_nobs.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_filtergenerate(inds,outref=filter); - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc - on macro entry) -) - -filename &outref temp; - -%if %mf_nobs(&inds)=0 %then %do; - /* ensure we have a default filter */ - data _null_; - file &outref; - put '1=1'; - run; -%end; -%else %do; - proc sort data=&inds; - by SUBGROUP_ID; - run; - data _null_; - file &outref lrecl=32800; - set &inds end=last; - by SUBGROUP_ID; - if _n_=1 then put '(('; - else if first.SUBGROUP_ID then put +1 GROUP_LOGIC '('; - else put +2 SUBGROUP_LOGIC; - - put +4 VARIABLE_NM OPERATOR_NM RAW_VALUE; - - if last.SUBGROUP_ID then put ')'@; - if last then put ')'; - run; -%end; - -%mend mp_filtergenerate; diff --git a/009_macros/mp_filterstore.sas b/009_macros/mp_filterstore.sas deleted file mode 100644 index a1cdd1a..0000000 --- a/009_macros/mp_filterstore.sas +++ /dev/null @@ -1,255 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks & Stores an input filter table and returns the Filter Key - @details Used to generate a FILTER_RK from an input query dataset. This - process requires several permanent tables (names are configurable). The - benefit of storing query values at backend is to enable stored 'views' of - filtered tables at frontend (ie, when building [SAS-Powered Apps]( - https://sasapps.io)). This macro is also used in [Data Controller for SAS]( - https://datacontroller.io). - - A more recent feature of this macro is the ability to support filter queries - on Format Catalogs. This is achieved by adding a `-FC` suffix to the `libds` - parameter - where the "ds" in this case is the catalog name. - - - @param [in] libds= The target dataset to be filtered (lib should be assigned). - If filtering a format catalog, add the following suffix: `-FC`. - @param [in] queryds= (WORK.FILTERQUERY) The temporary input query dataset to - be validated. Has the following format: -|GROUP_LOGIC:$3|SUBGROUP_LOGIC:$3|SUBGROUP_ID:8.|VARIABLE_NM:$32|OPERATOR_NM:$10|RAW_VALUE:$32767| -|---|---|---|---|---|---| -|AND|AND|1|SOME_BESTNUM|>|1| -|AND|AND|1|SOME_TIME|=|77333| - @param [in] filter_summary= (PERM.FILTER_SUMMARY) Permanent table containing - summary filter values. The definition is available by running - mp_coretable.sas as follows: `mp_coretable(FILTER_SUMMARY)`. Example - values: -|FILTER_RK:best.|FILTER_HASH:$32.|FILTER_TABLE:$41.|PROCESSED_DTTM:datetime19.| -|---|---|---|---| -|`1 `|`540E96F566D194AB58DD4C413C99C9DB `|`VIYA6014.MPE_TABLES `|`1956084246 `| -|`2 `|`87737DB9EEE2650F5C89956CEAD0A14F `|`VIYA6014.MPE_X_TEST `|`1956084452.1`| -|`3 `|`8048BD908DBBD83D013560734E90D394 `|`VIYA6014.MPE_TABLES `|`1956093620.6`| - @param [in] filter_detail= (PERM.FILTER_DETAIL) Permanent table containing - detailed (raw) filter values. The definition is available by running - mp_coretable.sas as follows: `mp_coretable(FILTER_DETAIL)`. Example - values: -|FILTER_HASH:$32.|FILTER_LINE:best.|GROUP_LOGIC:$3.|SUBGROUP_LOGIC:$3.|SUBGROUP_ID:best.|VARIABLE_NM:$32.|OPERATOR_NM:$12.|RAW_VALUE:$4000.|PROCESSED_DTTM:datetime19.| -|---|---|---|---|---|---|---|---|---| -|`540E96F566D194AB58DD4C413C99C9DB `|`1 `|`AND `|`AND `|`1 `|`LIBREF `|`CONTAINS `|`DC`|`1956084245.8 `| -|`540E96F566D194AB58DD4C413C99C9DB `|`2 `|`AND `|`OR `|`2 `|`DSN `|`= `|` MPE_LOCK_ANYTABLE `|`1956084245.8 `| -|`87737DB9EEE2650F5C89956CEAD0A14F `|`1 `|`AND `|`AND `|`1 `|`PRIMARY_KEY_FIELD `|`IN `|`(1,2,3) `|`1956084451.9 `| - @param [in] lock_table= (PERM.LOCK_TABLE) Permanent locking table. Used to - manage concurrent access. The definition is available by running - mp_coretable.sas as follows: `mp_coretable(LOCKTABLE)`. - @param [in] maxkeytable= (0) Optional permanent reference table used for - retained key tracking. Described in mp_retainedkey.sas. - @param [in] mdebug= (1) set to 1 to enable DEBUG messages - @param [out] outresult= (work.result) The result table with the FILTER_RK - @param [out] outquery= (work.query) The original query, taken as extract - after table load - - -

SAS Macros

- @li mddl_sas_cntlout.sas - @li mf_getuniquename.sas - @li mf_getvalue.sas - @li mf_islibds.sas - @li mf_nobs.sas - @li mp_abort.sas - @li mp_filtercheck.sas - @li mp_hashdataset.sas - @li mp_retainedkey.sas - -

Related Macros

- @li mp_filtercheck.sas - @li mp_filtergenerate.sas - @li mp_filtervalidate.sas - @li mp_filterstore.test.sas - - @version 9.2 - @author [Allan Bowe](https://www.linkedin.com/in/allanbowe) - -**//*** HELP END ***/ - -%macro mp_filterstore(libds=, - queryds=work.filterquery, - filter_summary=PERM.FILTER_SUMMARY, - filter_detail=PERM.FILTER_DETAIL, - lock_table=PERM.LOCK_TABLE, - maxkeytable=PERM.MAXKEYTABLE, - outresult=work.result, - outquery=work.query, - mdebug=1 -); -%put &sysmacroname entry vars:; -%put _local_; - -%local ds0 ds1 ds2 ds3 ds4 filter_hash orig_libds; -%let libds=%upcase(&libds); -%let orig_libds=&libds; - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=mp_filterstore - ,msg=%str(syscc=&syscc on macro entry) -) -%mp_abort(iftrue= (%mf_islibds(&filter_summary)=0) - ,mac=mp_filterstore - ,msg=%str(Invalid filter_summary value: &filter_summary) -) -%mp_abort(iftrue= (%mf_islibds(&filter_detail)=0) - ,mac=mp_filterstore - ,msg=%str(Invalid filter_detail value: &filter_detail) -) -%mp_abort(iftrue= (%mf_islibds(&lock_table)=0) - ,mac=mp_filterstore - ,msg=%str(Invalid lock_table value: &lock_table) -) - -/** - * validate query - * use format catalog export, if a format - */ -%if "%substr(&libds,%length(&libds)-2,3)"="-FC" %then %do; - %let libds=%scan(&libds,1,-); /* chop off -FC extension */ - %let ds0=%mf_getuniquename(prefix=fmtds_); - %let libds=&ds0; - /* - There is no need to export the entire format catalog here - the validations - are done against the data model, not the data values. So we can simply - hardcode the structure based on the cntlout dataset. - */ - %mddl_sas_cntlout(libds=&ds0) - -%end; -%mp_filtercheck(&queryds,targetds=&libds,abort=YES) - -/* hash the result */ -%let ds1=%mf_getuniquename(prefix=hashds); -%mp_hashdataset(&queryds,outds=&ds1,salt=&orig_libds) -%let filter_hash=%upcase(%mf_getvalue(&ds1,hashkey)); -%if &mdebug=1 %then %do; - data _null_; - putlog "filter_hash=&filter_hash"; - set &ds1; - putlog (_all_)(=); - run; -%end; - -/* check if data already exists for this hash */ -data &outresult; - set &filter_summary; - where filter_hash="&filter_hash"; -run; - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=mp_filterstore - ,msg=%str(syscc=&syscc after hash check) -) -%mp_abort(iftrue= ("&filter_hash "=" ") - ,mac=mp_filterstore - ,msg=%str(problem with filter_hash generation) -) - -%if %mf_nobs(&outresult)=0 %then %do; - - /* first update summary table */ - %let ds3=%mf_getuniquename(prefix=filtersum); - data work.&ds3; - if 0 then set &filter_summary; - filter_table="&orig_libds"; - filter_hash="&filter_hash"; - PROCESSED_DTTM=%sysfunc(datetime()); - output; - stop; - run; - - %mp_lockanytable(LOCK, - lib=%scan(&filter_summary,1,.) - ,ds=%scan(&filter_summary,2,.) - ,ref=MP_FILTERSTORE summary update - &filter_hash - ,ctl_ds=&lock_table - ) - - %let ds4=%mf_getuniquename(prefix=filtersumappend); - %mp_retainedkey( - base_lib=%scan(&filter_summary,1,.) - ,base_dsn=%scan(&filter_summary,2,.) - ,append_lib=work - ,append_dsn=&ds3 - ,retained_key=filter_rk - ,business_key=filter_hash - ,maxkeytable=&maxkeytable - ,locktable=&lock_table - ,outds=work.&ds4 - ) - proc append base=&filter_summary data=&ds4; - run; - - %mp_lockanytable(UNLOCK, - lib=%scan(&filter_summary,1,.) - ,ds=%scan(&filter_summary,2,.) - ,ref=MP_FILTERSTORE summary update - &filter_hash - ,ctl_ds=&lock_table - ) - - %if &syscc ne 0 %then %do; - data _null_; - set &ds4; - putlog (_all_)(=); - run; - %goto err; - %end; - - data &outresult; - set &filter_summary; - where filter_hash="&filter_hash"; - run; - - /* Next, update detail table */ - %let ds2=%mf_getuniquename(prefix=filterdetail); - data &ds2; - if 0 then set &filter_detail; - set &queryds; - format filter_hash $hex32. filter_line 8.; - filter_hash="&filter_hash"; - filter_line=_n_; - PROCESSED_DTTM=%sysfunc(datetime()); - run; - %mp_lockanytable(LOCK, - lib=%scan(&filter_detail,1,.) - ,ds=%scan(&filter_detail,2,.) - ,ref=MP_FILTERSTORE update - &filter_hash - ,ctl_ds=&lock_table - ) - proc append base=&filter_detail data=&ds2; - run; - - %mp_lockanytable(UNLOCK, - lib=%scan(&filter_detail,1,.) - ,ds=%scan(&filter_detail,2,.) - ,ref=MP_FILTERSTORE detail update &filter_hash - ,ctl_ds=&lock_table - ) - - %if &syscc ne 0 %then %do; - data _null_; - set &ds2; - putlog (_all_)(=); - run; - %goto err; - %end; - -%end; - -proc sort data=&filter_detail(where=(filter_hash="&filter_hash")) out=&outquery; - by filter_line; -run; - -%err: -%mp_abort(iftrue= (&syscc ne 0) - ,mac=mp_filterstore - ,msg=%str(syscc=&syscc on macro exit) -) - -%mend mp_filterstore; diff --git a/009_macros/mp_filtervalidate.sas b/009_macros/mp_filtervalidate.sas deleted file mode 100644 index 947d173..0000000 --- a/009_macros/mp_filtervalidate.sas +++ /dev/null @@ -1,105 +0,0 @@ -/*** HELP START ***//** - @file - @brief Checks a generated filter query for validity - @details Runs a generated filter in proc sql with the validate option. - Used in mp_filtercheck.sas in an fcmp container. - - Built to support dynamic filtering in - [Data Controller for SAS®](https://datacontroller.io). - - Usage: - - data work.filtertable; - infile datalines4 dsd; - input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. - OPERATOR_NM:$10. RAW_VALUE:$4000.; - datalines4; - AND,AND,1,AGE,=,12 - AND,AND,1,SEX,<=,"'M'" - AND,OR,2,Name,NOT IN,"('Jane','Alfred')" - AND,OR,2,Weight,>=,7 - ;;;; - run; - - %mp_filtergenerate(work.filtertable,outref=myfilter) - - %mp_filtervalidate(myfilter,sashelp.class) - - - @returns The SYSCC value will be 1008 if there are validation issues. - - @param [in] inref The input fileref to validate (generated by - mp_filtergenerate.sas) - @param [in] targetds The target dataset against which to verify the query - @param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions - @param [out] outds= (work.mp_filtervalidate) Output dataset containing the - err / warning message, if one exists. If this table contains any rows, - there are problems! - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_nobs.sas - @li mp_abort.sas - -

Related Macros

- @li mp_filtercheck.sas - @li mp_filtergenerate.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_filtervalidate(inref,targetds,abort=YES,outds=work.mp_filtervalidate); - -%mp_abort(iftrue= (&syscc ne 0 or &syserr ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc / syserr=&syserr - on macro entry) -) - -%local fref1; -%let fref1=%mf_getuniquefileref(); - -data _null_; - file &fref1; - infile &inref end=eof; - if _n_=1 then do; - put "proc sql;"; - put "validate select * from &targetds"; - put "where " ; - end; - input; - put _infile_; - putlog _infile_; - if eof then put ";quit;"; -run; - -%inc &fref1; - -data &outds; - if &sqlrc or &syscc or &syserr then do; - REASON_CD='VALIDATION_ERR'!!'OR: '!! - coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT')); - output; - end; - else stop; -run; - -filename &fref1 clear; - -%if %mf_nobs(&outds)>0 %then %do; - %if &abort=YES %then %do; - data _null_; - set &outds; - call symputx('REASON_CD',reason_cd,'l'); - stop; - run; - %mp_abort( - mac=&sysmacroname, - msg=%str(Filter validation issues.) - ) - %end; - %let syscc=1008; -%end; - -%mend mp_filtervalidate; diff --git a/009_macros/mp_getcols.sas b/009_macros/mp_getcols.sas deleted file mode 100644 index fb26036..0000000 --- a/009_macros/mp_getcols.sas +++ /dev/null @@ -1,68 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with column metadata. - @details This macro takes the `proc contents` output and "tidies it up" in the - following ways: - - @li Blank labels are filled in with column names - @li Formats are reconstructed with default values - @li Types such as DATE / TIME / DATETIME are inferred from the formats - - Example usage: - - %mp_getcols(sashelp.airline,outds=work.myds) - - @param [in] ds The dataset from which to obtain column metadata - @param [out] outds= (work.cols) The output dataset to create. Sample data: -|NAME:$32.|LENGTH:best.|VARNUM:best.|LABEL:$256.|FMTNAME:$32.|FORMAT:$49.|TYPE:$1.|DDTYPE:$9.| -|---|---|---|---|---|---|---|---| -|`AIR `|`8 `|`2 `|`international airline travel (thousands) `|` `|`8. `|`N `|`NUMERIC `| -|`DATE `|`8 `|`1 `|`DATE `|`MONYY `|`MONYY. `|`N `|`DATE `| -|`REGION `|`3 `|`3 `|`REGION `|` `|`$3. `|`C `|`CHARACTER `| - - -

Related Macros

- @li mf_getvarlist.sas - @li mm_getcols.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_getcols(ds, outds=work.cols); -%local dropds; -proc contents noprint data=&ds - out=_data_ (keep=name type length label varnum format:); -run; -%let dropds=&syslast; -data &outds(keep=name type length varnum format label ddtype fmtname); - set &dropds(rename=(format=fmtname type=type2)); - name=upcase(name); - if type2=2 then do; - length format $49.; - if fmtname='' then format=cats('$',length,'.'); - else if formatl=0 then format=cats(fmtname,'.'); - else format=cats(fmtname,formatl,'.'); - type='C'; - ddtype='CHARACTER'; - end; - else do; - if fmtname='' then format=cats(length,'.'); - else if formatl=0 then format=cats(fmtname,'.'); - else if formatd=0 then format=cats(fmtname,formatl,'.'); - else format=cats(fmtname,formatl,'.',formatd); - type='N'; - if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME'; - else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY' - or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA' - or format=:'MONYY' - then ddtype='DATE'; - else if format=:'TIME' then ddtype='TIME'; - else ddtype='NUMERIC'; - end; - if label='' then label=name; -run; -proc sql; -drop table &dropds; -%mend mp_getcols; diff --git a/009_macros/mp_getconstraints.sas b/009_macros/mp_getconstraints.sas deleted file mode 100644 index 734eb1c..0000000 --- a/009_macros/mp_getconstraints.sas +++ /dev/null @@ -1,115 +0,0 @@ -/*** HELP START ***//** - @file mp_getconstraints.sas - @brief Get constraint details at column level - @details Useful for capturing constraints before they are dropped / reapplied - during an update. - - proc sql; - create table work.example( - TX_FROM float format=datetime19., - DD_TYPE char(16), - DD_SOURCE char(2048), - DD_SHORTDESC char(256), - constraint pk primary key(tx_from, dd_type,dd_source), - constraint unq unique(tx_from, dd_type), - constraint nnn not null(DD_SHORTDESC) - ); - - %mp_getconstraints(lib=work,ds=example,outds=work.constraints) - - @param [in] lib= (WORK) The target library - @param [in] ds= The target dataset. Leave blank (default) for all datasets. - @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc - @param [out] outds= (mp_getconstraints) the output dataset - -

SAS Macros

- @li mf_getuniquename.sas - @li mp_dropmembers.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_getconstraints(lib=WORK - ,ds= - ,outds=mp_getconstraints - ,mdebug=0 -)/*/STORE SOURCE*/; - -%let lib=%upcase(&lib); -%let ds=%upcase(&ds); - -/** - * Cater for environments where sashelp.vcncolu is not available - */ -%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do; - proc sql; - create table &outds( - libref char(8) - ,TABLE_NAME char(32) - ,constraint_type char(8) label='Constraint Type' - ,constraint_name char(32) label='Constraint Name' - ,column_name char(32) label='Column' - ,constraint_order num - ); - %return; -%end; - -/** - * Neither dictionary tables nor sashelp provides a constraint order column, - * however they DO arrive in the correct order. So, create the col. - **/ -%local vw; -%let vw=%mf_getuniquename(prefix=mp_getconstraints_vw_); -data &vw /view=&vw; - set sashelp.vcncolu; - where table_catalog="&lib"; - - /* use retain approach to reset the constraint order with each constraint */ - length tmp $1000; - retain tmp; - drop tmp; - if tmp ne catx('|',table_catalog,table_name,constraint_name) then do; - constraint_order=1; - end; - else constraint_order+1; - tmp=catx('|',table_catalog, table_name,constraint_name); -run; - -/* must use SQL as proc datasets does not support length changes */ -proc sql noprint; -create table &outds as - select upcase(a.TABLE_CATALOG) as libref - ,upcase(a.TABLE_NAME) as TABLE_NAME - ,a.constraint_type - ,a.constraint_name - ,b.column_name - ,b.constraint_order - from dictionary.TABLE_CONSTRAINTS a - left join &vw b - on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG) - and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME) - and a.constraint_name=b.constraint_name -/** - * We cannot apply this clause to the underlying dictionary table. See: - * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867 - * cannot use`where calculated libref="&lib"` either as it will STILL execute - * all the underlying constraint queries, causing exception errors in some - * cases: https://github.com/sasjs/core/issues/283 - */ - where a.TABLE_CATALOG="&lib" - %if "&ds" ne "" %then %do; - and upcase(a.TABLE_NAME)="&ds" - and upcase(b.TABLE_NAME)="&ds" - %end; - order by libref, table_name, constraint_name, constraint_order - ; - -/* tidy up */ -%mp_dropmembers( - &vw, - iftrue=(&mdebug=0) -) - -%mend mp_getconstraints; diff --git a/009_macros/mp_getdbml.sas b/009_macros/mp_getdbml.sas deleted file mode 100644 index 292d5a9..0000000 --- a/009_macros/mp_getdbml.sas +++ /dev/null @@ -1,335 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract DBML from SAS Libraries - @details DBML is an open source markup format to represent databases. - More details: https://www.dbml.org/home/ - - Usage: - - - %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES) - - Take the log output and paste it into the renderer at https://dbdiagram.io - to view your data model diagram. The code takes a "best guess" at - the one to one and one to many relationships (based on constraints - and indexes, and assuming that the column names would match). - - You may need to adjust the rendered DBML to suit your needs. - - ![dbml for sas](https://i.imgur.com/8T1tIZp.gif) - - -

SAS Macros

- @li mf_getquotedstr.sas - @li mp_getconstraints.sas - - @param [in] liblist= (SASHELP) Space seperated list of librefs to take as - input - @param [out] outref= (getdbml) Fileref to contain the DBML - @param [in] showlog= (NO) set to YES to show the DBML in the log - - @version 9.3 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO -)/*/STORE SOURCE*/; - -/* check fileref is assigned */ -%if %sysfunc(fileref(&outref)) > 0 %then %do; - filename &outref temp; -%end; - -%let liblist=%upcase(&liblist); - -proc sql noprint; -create table _data_ as - select * from dictionary.tables - where upcase(libname) in (%mf_getquotedstr(&liblist)) - order by libname,memname; -%local tabinfo; %let tabinfo=&syslast; - -create table _data_ as - select * from dictionary.columns - where upcase(libname) in (%mf_getquotedstr(&liblist)) - order by libname,memname,varnum; -%local colinfo; %let colinfo=&syslast; - -%local dsnlist; - select distinct upcase(cats(libname,'.',memname)) into: dsnlist - separated by ' ' - from &syslast -; - -create table _data_ as - select * from dictionary.indexes - where upcase(libname) in (%mf_getquotedstr(&liblist)) - order by idxusage, indxname, indxpos; -%local idxinfo; %let idxinfo=&syslast; - -/* Extract all Primary Key and Unique data constraints */ -%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_) -%local colconst; %let colconst=&syslast; - -%do x=2 %to %sysfunc(countw(&liblist)); - %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_) - proc append base=&colconst data=&syslast; - run; -%end; - - - - -/* header info */ -data _null_; - file &outref; - put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) "; - put "Project sasdbml {"; - put " database_type: 'SAS'"; - put " Note: 'Generated by the mp_getdbml() macro'"; - put "}"; -run; - -/* create table groups */ -data _null_; - file &outref mod; - set &tabinfo; - by libname; - if first.libname then put "TableGroup " libname "{"; - ds=quote(cats(libname,'.',memname)); - put ' ' ds; - if last.libname then put "}"; -run; - -/* table for pks */ -data _data_; - length curds const col $39; - call missing (of _all_); - stop; -run; -%let pkds=&syslast; - -%local x curds constraints_used constcheck; -%do x=1 %to %sysfunc(countw(&dsnlist,%str( ))); - %let curds=%scan(&dsnlist,&x,%str( )); - %let constraints_used=; - %let constcheck=0; - data _null_; - file &outref mod; - length lab $1024 typ $20; - set &colinfo (where=( - libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)" - )) end=last; - - if _n_=1 then do; - table='Table "'!!"&curds"!!'"{'; - put table; - end; - name=upcase(name); - lab=" note:"!!quote(trim(tranwrd(label,'"',"'"))); - if upcase(format)=:'DATETIME' then typ='datetime'; - else if type='char' then typ=cats('char(',length,')'); - else typ='num'; - - if notnull='yes' then notnul=' not null'; - if notnull='no' and missing(label) then put ' ' name typ; - else if notnull='yes' and missing(label) then do; - put ' ' name typ '[' notnul ']'; - end; - else if notnull='no' then put ' ' name typ '[' lab ']'; - else put ' ' name typ '[' notnul ',' lab ']'; - - run; - - data _data_(keep=curds const col); - length ctype $11 cols constraints_used $5000; - set &colconst (where=( - upcase(libref)="%scan(&curds,1,.)" - and upcase(table_name)="%scan(&curds,2,.)" - and constraint_type in ('PRIMARY','UNIQUE') - )) end=last; - file &outref mod; - by constraint_type constraint_name; - retain cols; - column_name=upcase(column_name); - - if _n_=1 then put / ' indexes {'; - - if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]'; - else ctype='[unique]'; - - if first.constraint_name then cols = cats('(',column_name); - else cols=cats(cols,',',column_name); - - if last.constraint_name then do; - cols=cats(cols,')',ctype)!!' //'!!constraint_name; - put ' ' cols; - constraints_used=catx(' ',constraints_used, constraint_name); - call symputx('constcheck',1); - end; - - if last then call symput('constraints_used',cats(upcase(constraints_used))); - - length curds const col $39; - curds="&curds"; - const=constraint_name; - col=column_name; - run; - - proc append base=&pkds data=&syslast;run; - - /* Create Unique Indexes, but only if they were not already defined within - the Constraints section. */ - data _data_(keep=curds const col); - set &idxinfo (where=( - libname="%scan(&curds,1,.)" - and upcase(memname)="%scan(&curds,2,.)" - and unique='yes' - and upcase(indxname) not in (%mf_getquotedstr(&constraints_used)) - )); - file &outref mod; - by idxusage indxname; - name=upcase(name); - if &constcheck=1 then stop; /* we only care about PKs so stop if we have */ - if _n_=1 and &constcheck=0 then put / ' indexes {'; - - length cols $5000; - retain cols; - if first.indxname then cols = cats('(',name); - else cols=cats(cols,',',name); - - if last.indxname then do; - cols=cats(cols,')[unique]')!!' //'!!indxname; - put ' ' cols; - call symputx('constcheck',1); - end; - - length curds const col $39; - curds="&curds"; - const=indxname; - col=name; - run; - proc append base=&pkds data=&syslast;run; - - data _null_; - file &outref mod; - if &constcheck =1 then put ' }'; - put '}'; - run; - -%end; - -/** - * now we need to figure out the relationships - */ - -/* sort alphabetically so we can have one set of unique cols per table */ -proc sort data=&pkds nodupkey; - by curds const col; -run; - -data &pkds.1 (keep=curds col) - &pkds.2 (keep=curds cols); - set &pkds; - by curds const; - length retconst $39 cols $5000; - retain retconst cols; - if first.curds then do; - retconst=const; - cols=upcase(col); - end; - else cols=catx(' ',cols,upcase(col)); - if retconst=const then do; - output &pkds.1; - if last.const then output &pkds.2; - end; -run; - -%let curdslist="0"; -%do x=1 %to %sysfunc(countw(&dsnlist,%str( ))); - %let curds=%scan(&dsnlist,&x,%str( )); - - %let pkcols=0; - data _null_; - set &pkds.2(where=(curds="&curds")); - call symputx('pkcols',cols); - run; - %if &pkcols ne 0 %then %do; - %let curdslist=&curdslist,"&curds"; - - /* start with one2one */ - data &pkds.4; - file &outref mod; - set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist))); - line='Ref: "'!!"&curds" - !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')') - !!' - ' - !!cats(quote(trim(curds)) - ,'.(' - ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))" - ,')' - ); - put line; - run; - - /* now many2one */ - /* get table with one row per col */ - data &pkds.5; - set &pkds.1(where=(curds="&curds")); - run; - /* get tables which contain the PK columns */ - proc sql; - create table &pkds.5a as - select upcase(cats(b.libname,'.',b.memname)) as curds - ,b.name - from &pkds.5 a - inner join &colinfo b - on a.col=upcase(b.name); - /* count to make sure those tables contain ALL the columns */ - create table &pkds.5b as - select curds,count(*) as cnt - from &pkds.5a - where curds not in ( - select curds from &pkds.2 where cols="&pkcols" - ) /* not a one to one match */ - and curds ne "&curds" /* exclude self */ - group by 1; - create table &pkds.6 as - select a.* - ,b.cols - from &pkds.5b a - left join &pkds.4 b - on a.curds=b.curds; - - data _null_; - set &pkds.6; - file &outref mod; - colcnt=%sysfunc(countw(&pkcols)); - if cnt=colcnt then do; - /* table contains all the PK cols, and was not a direct / 121 match */ - line='Ref: "'!!"&curds" - !!'".(' - !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))" - !!') > ' - !!cats(quote(trim(curds)) - ,'.(' - ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))" - ,')' - ); - put line; - end; - run; - %end; -%end; - - -%if %upcase(&showlog)=YES %then %do; - options ps=max; - data _null_; - infile &outref; - input; - putlog _infile_; - run; -%end; - -%mend mp_getdbml; diff --git a/009_macros/mp_getddl.sas b/009_macros/mp_getddl.sas deleted file mode 100644 index 6bbf8f0..0000000 --- a/009_macros/mp_getddl.sas +++ /dev/null @@ -1,405 +0,0 @@ -/*** HELP START ***//** - @file mp_getddl.sas - @brief Extract DDL in various formats, by table or library - @details Data Definition Language relates to a set of SQL instructions used - to create tables in SAS or a database. The macro can be used at table or - library level. The default behaviour is to create DDL in SAS format. - - Note - views are not currently supported. - - Usage: - - data test(index=(pk=(x y)/unique /nomiss)); - x=1; - y='blah'; - label x='blah'; - run; - proc sql; describe table &syslast; - %mp_getddl(work,test,flavour=tsql,showlog=YES) - -

SAS Macros

- @li mf_existfileref.sas - @li mf_getvarcount.sas - @li mp_getconstraints.sas - - @param [in] libref Libref of the library to create DDL for. Should already - be assigned. - @param [in] ds dataset to create ddl for (optional) - @param [in] fref= (getddl) the fileref to which to _append_ the DDL. If it - does not exist, it will be created. - @param [in] flavour= (SAS) The type of DDL to create. Options: - @li SAS - @li TSQL - - @param [in]showlog= (NO) Set to YES to show the DDL in the log - @param [in] schema= () Choose a preferred schema name (default is to use - actual schema, else libref) - @param [in] applydttm= (NO) For non SAS DDL, choose if columns are created - with native datetime2 format or regular decimal type - - @version 9.3 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_getddl(libref,ds,fref=getddl,flavour=SAS,showlog=NO,schema= - ,applydttm=NO -)/*/STORE SOURCE*/; - -/* check fileref is assigned */ -%if %mf_existfileref(&fref)=0 %then %do; - filename &fref temp ; -%end; - -%if %length(&libref)=0 %then %let libref=WORK; -%let flavour=%upcase(&flavour); - -proc sql noprint; -create table _data_ as - select * from dictionary.tables - where upcase(libname)="%upcase(&libref)" - and memtype='DATA' /* views not currently supported */ - %if %length(&ds)>0 %then %do; - and upcase(memname)="%upcase(&ds)" - %end; - ; -%local tabinfo; %let tabinfo=&syslast; - -create table _data_ as - select * from dictionary.columns - where upcase(libname)="%upcase(&libref)" - %if %length(&ds)>0 %then %do; - and upcase(memname)="%upcase(&ds)" - %end; - ; -%local colinfo; %let colinfo=&syslast; - -%local dsnlist; - select distinct upcase(memname) into: dsnlist - separated by ' ' - from &syslast -; - -create table _data_ as - select * from dictionary.indexes - where upcase(libname)="%upcase(&libref)" - %if %length(&ds)>0 %then %do; - and upcase(memname)="%upcase(&ds)" - %end; - order by idxusage, indxname, indxpos - ; -%local idxinfo; %let idxinfo=&syslast; - -/* Extract all Primary Key and Unique data constraints */ -%mp_getconstraints(lib=%upcase(&libref),ds=%upcase(&ds),outds=_data_) -%local colconst; %let colconst=&syslast; - -%macro addConst(); - %global constraints_used; - data _null_; - length ctype $11 constraint_name_orig $256 constraints_used $5000; - set &colconst( - where=(table_name="&curds" and constraint_type in ('PRIMARY','UNIQUE')) - ) end=last; - file &fref mod; - by constraint_type constraint_name; - retain constraints_used; - constraint_name_orig=constraint_name; - if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='PRIMARY KEY'; - else ctype=strip(constraint_type); - %if &flavour=TSQL %then %do; - column_name=catt('[',column_name,']'); - constraint_name=catt('[',constraint_name,']'); - %end; - %else %if &flavour=PGSQL %then %do; - column_name=catt('"',column_name,'"'); - constraint_name=catt('"',constraint_name,'"'); - %end; - if first.constraint_name then do; - constraints_used = catx(' ', constraints_used, constraint_name_orig); - put " ,CONSTRAINT " constraint_name ctype "(" ; - put ' ' column_name; - end; - else put ' ,' column_name; - if last.constraint_name then do; - put " )"; - call symput('constraints_used',strip(constraints_used)); - end; - run; - %put &=constraints_used; -%mend addConst; - -data _null_; - file &fref mod; - put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */"; -run; - -%local x curds; -%if &flavour=SAS %then %do; - %do x=1 %to %sysfunc(countw(&dsnlist)); - %let curds=%scan(&dsnlist,&x); - data _null_; - file &fref mod; - put "/* SAS Flavour DDL for %upcase(&libref).&curds */"; - put "proc sql;"; - run; - data _null_; - file &fref mod; - length lab $1024 typ $20; - set &colinfo (where=(upcase(memname)="&curds")) end=last; - - if _n_=1 then do; - if memtype='DATA' then do; - put "create table &libref..&curds("; - end; - else do; - /* just a placeholder - we filter out views at the top */ - put "create view &libref..&curds("; - end; - put " "@@; - end; - else put " ,"@@; - if length(format)>1 then fmt=" format="!!cats(format); - if length(label)>1 then - lab=" label="!!cats("'",tranwrd(label,"'","''"),"'"); - if notnull='yes' then notnul=' not null'; - if type='char' then typ=cats('char(',length,')'); - else if length ne 8 then typ='num length='!!cats(length); - else typ='num'; - put name typ fmt notnul lab; - run; - - /* Extra step for data constraints */ - %addConst() - - data _null_; - file &fref mod; - put ');'; - run; - - /* Create Unique Indexes, but only if they were not already defined within - the Constraints section. */ - data _null_; - *length ds $128; - set &idxinfo( - where=( - memname="&curds" - and unique='yes' - and indxname not in ( - %sysfunc(tranwrd("&constraints_used",%str( ),%str(","))) - ) - ) - ); - file &fref mod; - by idxusage indxname; -/* ds=cats(libname,'.',memname); */ - if first.indxname then do; - put 'CREATE UNIQUE INDEX ' indxname "ON &libref..&curds (" ; - put ' ' name ; - end; - else put ' ,' name ; - *else put ' ,' name ; - if last.indxname then do; - put ');'; - end; - run; - -/* - ods output IntegrityConstraints=ic; - proc contents data=testali out2=info; - run; - */ - %end; -%end; -%else %if &flavour=TSQL %then %do; - /* if schema does not exist, set to be same as libref */ - %local schemaactual; - proc sql noprint; - select sysvalue into: schemaactual - from dictionary.libnames - where upcase(libname)="&libref" and engine='SQLSVR'; - %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref)); - - %do x=1 %to %sysfunc(countw(&dsnlist)); - %let curds=%scan(&dsnlist,&x); - data _null_; - file &fref mod; - put "/* TSQL Flavour DDL for &schema..&curds */"; - data _null_; - file &fref mod; - set &colinfo (where=(upcase(memname)="&curds")) end=last; - if _n_=1 then do; - if memtype='DATA' then do; - put "create table [&schema].[&curds]("; - end; - else do; - /* just a placeholder - we filter out views at the top */ - put "create view [&schema].[&curds]("; - end; - put " "@@; - end; - else put " ,"@@; - format=upcase(format); - if 1=0 then; /* dummy if */ - %if &applydttm=YES %then %do; - else if format=:'DATETIME' then fmt='[datetime2](7) '; - %end; - else if type='num' then fmt='[decimal](18,2)'; - else if length le 8000 then fmt='[varchar]('!!cats(length)!!')'; - else fmt=cats('[varchar](max)'); - if notnull='yes' then notnul=' NOT NULL'; - put "[" name +(-1) "]" fmt notnul; - run; - - /* Extra step for data constraints */ - %addConst() - - /* Create Unique Indexes, but only if they were not already defined within - the Constraints section. */ - data _null_; - *length ds $128; - set &idxinfo( - where=( - memname="&curds" - and unique='yes' - and indxname not in ( - %sysfunc(tranwrd("&constraints_used",%str( ),%str(","))) - ) - ) - ); - file &fref mod; - by idxusage indxname; - *ds=cats(libname,'.',memname); - if first.indxname then do; - /* add nonclustered in case of multiple unique indexes */ - put ' ,index [' indxname +(-1) '] UNIQUE NONCLUSTERED ('; - put ' [' name +(-1) ']'; - end; - else put ' ,[' name +(-1) ']'; - if last.indxname then do; - put ' )'; - end; - run; - - data _null_; - file &fref mod; - put ')'; - put 'GO'; - run; - - /* add extended properties for labels */ - data _null_; - file &fref mod; - length nm $64 lab $1024; - set &colinfo (where=(upcase(memname)="&curds" and label ne '')) end=last; - nm=cats("N'",tranwrd(name,"'","''"),"'"); - lab=cats("N'",tranwrd(label,"'","''"),"'"); - put ' '; - put "EXEC sys.sp_addextendedproperty "; - put " @name=N'MS_Description',@value=" lab ; - put " ,@level0type=N'SCHEMA',@level0name=N'&schema' "; - put " ,@level1type=N'TABLE',@level1name=N'&curds'"; - put " ,@level2type=N'COLUMN',@level2name=" nm ; - if last then put 'GO'; - run; - %end; -%end; -%else %if &flavour=PGSQL %then %do; - /* if schema does not exist, set to be same as libref */ - %local schemaactual; - proc sql noprint; - select sysvalue into: schemaactual - from dictionary.libnames - where upcase(libname)="&libref" and engine='POSTGRES'; - %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref)); - data _null_; - file &fref mod; - put "CREATE SCHEMA &schema;"; - %do x=1 %to %sysfunc(countw(&dsnlist)); - %let curds=%scan(&dsnlist,&x); - %local curdsvarcount; - %let curdsvarcount=%mf_getvarcount(&libref..&curds); - %if &curdsvarcount>1600 %then %do; - data _null_; - file &fref mod; - put "/* &libref..&curds contains &curdsvarcount vars */"; - put "/* Postgres cannot create tables with over 1600 vars */"; - put "/* No DDL will be generated for this table"; - run; - %end; - %else %do; - data _null_; - file &fref mod; - put "/* Postgres Flavour DDL for &schema..&curds */"; - data _null_; - file &fref mod; - set &colinfo (where=(upcase(memname)="&curds")) end=last; - length fmt $32; - if _n_=1 then do; - if memtype='DATA' then do; - put "CREATE TABLE &schema..&curds ("; - end; - else do; - /* just a placeholder - we filter out views at the top */ - put "CREATE VIEW &schema..&curds ("; - end; - put " "@@; - end; - else put " ,"@@; - format=upcase(format); - if 1=0 then; /* dummy if */ - %if &applydttm=YES %then %do; - else if format=:'DATETIME' then fmt=' TIMESTAMP '; - %end; - else if type='num' then fmt=' DOUBLE PRECISION'; - else fmt='VARCHAR('!!cats(length)!!')'; - if notnull='yes' then notnul=' NOT NULL'; - /* quote column names in case they represent reserved words */ - name2=quote(trim(name)); - put name2 fmt notnul; - run; - - /* Extra step for data constraints */ - %addConst() - - data _null_; - file &fref mod; - put ');'; - run; - - /* Create Unique Indexes, but only if they were not already defined within - the Constraints section. */ - data _null_; - *length ds $128; - set &idxinfo( - where=( - memname="&curds" - and unique='yes' - and indxname not in ( - %sysfunc(tranwrd("&constraints_used",%str( ),%str(","))) - ) - ) - ); - file &fref mod; - by idxusage indxname; - if first.indxname then do; - put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds("; - put ' "' name +(-1) '"' ; - end; - else put ' ,"' name +(-1) '"'; - if last.indxname then do; - put ');'; - end; - run; - %end; - %end; -%end; -%if %upcase(&showlog)=YES %then %do; - options ps=max; - data _null_; - infile &fref; - input; - putlog _infile_; - run; -%end; - -%mend mp_getddl; diff --git a/009_macros/mp_getformats.sas b/009_macros/mp_getformats.sas deleted file mode 100644 index 974e1a6..0000000 --- a/009_macros/mp_getformats.sas +++ /dev/null @@ -1,134 +0,0 @@ -/*** HELP START ***//** - @file - @brief Export format definitions - @details Formats are exported from the first (if any) catalog entry in the - FMTSEARCH path. - - Formats are taken from the library / dataset reference and / or a static - format list. - - Note - the source for this information is the dictionary.formats table. This - cannot show formats that are not already declared in the FMTSEARCH path. - - Example usage: - - %mp_getformats(lib=sashelp,ds=prdsale,outsummary=work.dictable) - - %mp_getformats(fmtlist=FORMAT1 $FORMAT2 @INFMT3,outsummary=work.table2) - - @param [in] lib= (0) The libref for which to return formats. - @todo Enable exporting of formats for an entire library - @param [in] ds= (0) The dataset from which to obtain format definitions - @param [in] fmtlist= (0) A list of additional format names - @param [out] outsummary= (work.mp_getformats_summary) Output dataset - containing summary definitions - structure taken from dictionary.formats as - follows: - - |libname:$8.|memname:$32.|path:$1024.|objname:$32.|fmtname:$32.|fmttype:$1.|source:$1.|minw:best.|mind:best.|maxw:best.|maxd:best.|defw:best.|defd:best.| - |---|---|---|---|---|---|---|---|---|---|---|---|---| - | | | | |$|F|B|1|0|32767|0|1|0| - | | | | |$|I|B|1|0|32767|0|1|0| - |` `|` `|/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIANYDT|$ANYDTIF|I|U|1|0|60|0|19|0| - | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFASCII|$ASCII|F|U|1|0|32767|0|1|0| - | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWIASCII|$ASCII|I|U|1|0|32767|0|1|0| - | | |/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe|UWFBASE6|$BASE64X|F|U|1|0|32767|0|1|0| - - - @param [out] outdetail= (0) Provide an output dataset in which to export all - the custom format definitions (from proc format CNTLOUT). Definitions: -https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#a002473477.htm - Sample data: - - |FMTNAME:$32.|START:$16.|END:$16.|LABEL:$256.|MIN:best.|MAX:best.|DEFAULT:best.|LENGTH:best.|FUZZ:best.|PREFIX:$2.|MULT:best.|FILL:$1.|NOEDIT:best.|TYPE:$1.|SEXCL:$1.|EEXCL:$1.|HLO:$13.|DECSEP:$1.|DIG3SEP:$1.|DATATYPE:$8.|LANGUAGE:$8.| - |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| - |`WHICHPATH `|`0 `|`0 `|`path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|` `|` `|` `|` `|` `| - |`WHICHPATH `|`**OTHER** `|`**OTHER** `|`big fat problem if not path1 `|`1 `|`40 `|`28 `|`28 `|`1E-12 `|` `|`0 `|` `|`0 `|`N `|`N `|`N `|`O `|` `|` `|` `|` `| - -

SAS Macros

- @li mddl_sas_cntlout.sas - @li mf_dedup.sas - @li mf_getfmtlist.sas - @li mf_getfmtname.sas - @li mf_getquotedstr.sas - @li mf_getuniquename.sas - - -

Related Macros

- @li mf_getfmtlist.sas - @li mp_applyformats.sas - @li mp_cntlout.sas - @li mp_getformats.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_getformats(lib=0 - ,ds=0 - ,fmtlist=0 - ,outsummary=work.mp_getformats_summary - ,outdetail=0 -); - -%local i fmt allfmts tempds fmtcnt; - -%if "&fmtlist" ne "0" %then %do i=1 %to %sysfunc(countw(&fmtlist,%str( ))); - /* ensure format list contains format _name_ only */ - %let fmt=%scan(&fmtlist,&i,%str( )); - %let fmt=%mf_getfmtname(&fmt); - %let allfmts=&allfmts &fmt; -%end; - -%if &ds=0 and &lib ne 0 %then %do; - /* grab formats from library */ - /* to do */ -%end; -%else %if &ds ne 0 and &lib ne 0 %then %do; - /* grab formats from dataset */ - %let allfmts=%mf_getfmtlist(&lib..&ds) &allfmts; -%end; - -/* ensure list is unique */ -%let allfmts=%mf_dedup(%upcase(&allfmts)); - -/* create summary table */ -%if %index(&outsummary,.)=0 %then %let outsummary=WORK.&outsummary; -proc sql; -create table &outsummary as - select * from dictionary.formats - where fmtname in (%mf_getquotedstr(&allfmts,quote=D)); - -%if "&outdetail" ne "0" %then %do; - /* ensure base table always exists */ - %mddl_sas_cntlout(libds=&outdetail) - /* grab the location of each format */ - %let fmtcnt=0; - data _null_; - set &outsummary; - if not missing(libname); - x+1; - call symputx(cats('fmtloc',x),cats(libname,'.',memname),'l'); - call symputx(cats('fmtname',x),fmtname,'l'); - call symputx('fmtcnt',x,'l'); - run; - /* export each format and append to the output table */ - %let tempds=%mf_getuniquename(prefix=mp_getformats); - %do i=1 %to &fmtcnt; - proc format library=&&fmtloc&i CNTLOUT=&tempds; - select &&fmtname&i; - run; - data &tempds; - if 0 then set &outdetail; - set &tempds; - /* set fmtrow (position of record within the format) */ - by type fmtname notsorted; - if first.fmtname then fmtrow=1; - else fmtrow+1; - run; - proc append base=&outdetail data=&tempds ; - run; - %end; -%end; - -%mend mp_getformats; diff --git a/009_macros/mp_getmaxvarlengths.sas b/009_macros/mp_getmaxvarlengths.sas deleted file mode 100644 index aff390a..0000000 --- a/009_macros/mp_getmaxvarlengths.sas +++ /dev/null @@ -1,130 +0,0 @@ -/*** HELP START ***//** - @file - @brief Scans a dataset to find the max length of the variable values - @details - This macro will scan a base dataset and produce an output dataset with two - columns: - - - NAME Name of the base dataset column - - MAXLEN Maximum length of the data contained therein. - - Character fields are often allocated very large widths (eg 32000) of which the - maximum value is likely to be much narrower. Identifying such cases can be - helpful in the following scenarios: - - @li Enabling a HTML table to be appropriately sized (`num2char=YES`) - @li Reducing the size of a dataset to save on storage (mp_ds2squeeze.sas) - @li Identifying columns containing nothing but missing values (`MAXLEN=0` in - the output table) - - If the entire column is made up of (non-special) missing values then a value - of 0 is returned. - - Usage: - - %mp_getmaxvarlengths(sashelp.class,outds=work.myds) - - @param [in] libds Two part dataset (or view) reference. - @param [in] num2char= (NO) When set to NO, numeric fields are sized according - to the number of bytes used (or set to zero in the case of non-special - missings). When YES, the numeric field is converted to character (using the - format, if available), and that is sized instead, using `lengthn()`. - @param [out] outds= The output dataset to create, eg: - |NAME:$8.|MAXLEN:best.| - |---|---| - |`Name `|`7 `| - |`Sex `|`1 `| - |`Age `|`3 `| - |`Height `|`8 `| - |`Weight `|`3 `| - -

SAS Macros

- @li mcf_length.sas - @li mf_getuniquename.sas - @li mf_getvarcount.sas - @li mf_getvarlist.sas - @li mf_getvartype.sas - @li mf_getvarformat.sas - - @version 9.2 - @author Allan Bowe - -

Related Macros

- @li mp_ds2squeeze.sas - @li mp_getmaxvarlengths.test.sas - -**//*** HELP END ***/ - -%macro mp_getmaxvarlengths( - libds - ,num2char=NO - ,outds=work.mp_getmaxvarlengths -)/*/STORE SOURCE*/; - -%local vars prefix x var fmt srcds; -%let vars=%mf_getvarlist(libds=&libds); -%let prefix=%substr(%mf_getuniquename(),1,25); -%let num2char=%upcase(&num2char); - -%if &num2char=NO %then %do; - /* compile length function for numeric fields */ - %mcf_length(wrap=YES, insert_cmplib=YES) -%end; - -%if &num2char=NO - and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5") - and %mf_getvarcount(&libds,typefilter=N) gt 0 -%then %do; - /* custom functions not supported in summary operations */ - %let srcds=%mf_getuniquename(); - data &srcds/view=&srcds; - set &libds; - %do x=1 %to %sysfunc(countw(&vars,%str( ))); - %let var=%scan(&vars,&x); - %if %mf_getvartype(&libds,&var)=N %then %do; - &prefix.&x=mcf_length(&var); - %end; - %end; - run; -%end; -%else %let srcds=&libds; - -proc sql; -create table &outds (rename=( - %do x=1 %to %sysfunc(countw(&vars,%str( ))); - &prefix.&x=%scan(&vars,&x) - %end; - )) - as select - %do x=1 %to %sysfunc(countw(&vars,%str( ))); - %let var=%scan(&vars,&x); - %if &x>1 %then ,; - %if %mf_getvartype(&libds,&var)=C %then %do; - max(lengthn(&var)) as &prefix.&x - %end; - %else %if &num2char=YES %then %do; - %let fmt=%mf_getvarformat(&libds,&var); - %put fmt=&fmt; - %if %str(&fmt)=%str() %then %do; - max(lengthn(cats(&var))) as &prefix.&x - %end; - %else %do; - max(lengthn(put(&var,&fmt))) as &prefix.&x - %end; - %end; - %else %do; - %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; - max(&prefix.&x) as &prefix.&x - %end; - %else %do; - max(mcf_length(&var)) as &prefix.&x - %end; - %end; - %end; - from &srcds; - - proc transpose data=&outds - out=&outds(rename=(_name_=NAME COL1=MAXLEN)); - run; - -%mend mp_getmaxvarlengths; diff --git a/009_macros/mp_getpk.sas b/009_macros/mp_getpk.sas deleted file mode 100644 index 05e6da4..0000000 --- a/009_macros/mp_getpk.sas +++ /dev/null @@ -1,281 +0,0 @@ -/*** HELP START ***//** - @file - @brief Extract the primary key fields from a table or library - @details Examines the constraints to identify primary key fields - indicated - by an explicit PK constraint, or a unique index that is also NOT NULL. - - Can be executed at both table and library level. Supports both BASE engine - libraries and SQL Server. - - Usage: - - proc sql; - create table work.example( - TX_FROM float format=datetime19., - DD_TYPE char(16), - DD_SOURCE char(2048), - DD_SHORTDESC char(256), - constraint pk primary key(tx_from, dd_type,dd_source), - constraint unq unique(tx_from, dd_type), - constraint nnn not null(DD_SHORTDESC) - ); - %mp_getpk(work,ds=example) - - Returns: - -|libref:$8.|dsn:$32.|memtype:$8.|dbms_memtype:$32.|typemem:$8.|memlabel:$256.|nvar:best.|compress:$8.|pk_fields:$512.| -|---|---|---|---|---|---|---|---|---| -|WORK|EXAMPLE|DATA| |DATA| |4|NO|TX_FROM DD_TYPE DD_SOURCE| - - - @param [in] lib The libref to examine - @param [in] ds= (0) Select the dataset to examine, else use 0 for all tables - @param [in] mdebug= (0) Set to 1 to preserve temp tables, print var values etc - @param [out] outds= (work.mp_getpk) The name of the output table to create. - -

SAS Macros

- @li mf_existfeature.sas - @li mf_getengine.sas - @li mf_getschema.sas - @li mp_dropmembers.sas - @li mp_getconstraints.sas - -

Related Macros

- @li mp_getpk.test.sas - @li mp_guesspk.sas - - @version 9.3 - @author Macro People Ltd -**//*** HELP END ***/ - -%macro mp_getpk( - lib, - ds=0, - outds=work.mp_getpk, - mdebug=0 -)/*/STORE SOURCE*/; - - -%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks - pkfromindex; - -%let lib=%upcase(&lib); -%let ds=%upcase(&ds); -%let engine=%mf_getengine(&lib); -%let schema=%mf_getschema(&lib); - -%let ds1=%mf_getuniquename(prefix=getpk_ds1); -%let ds2=%mf_getuniquename(prefix=getpk_ds2); -%let ds3=%mf_getuniquename(prefix=getpk_ds3); -%let tabs1=%mf_getuniquename(prefix=getpk_tabs1); -%let tabs2=%mf_getuniquename(prefix=getpk_tabs2); -%let sum=%mf_getuniquename(prefix=getpk_sum); -%let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure); -%let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault); -%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex); -%let finalpks=%mf_getuniquename(prefix=getpk_finalpks); - -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -proc sql; -create table &ds1 as - select libname as libref - ,upcase(memname) as dsn - ,memtype - ,upcase(name) as name - ,type - ,length - ,varnum - ,label - ,format - ,idxusage - ,notnull - from dictionary.columns - where upcase(libname)="&lib" -%if &ds ne 0 %then %do; - and upcase(memname)="&ds" -%end; - ; - - -%if &engine=SQLSVR %then %do; - proc sql; - connect using &lib; - create table work.&ds2 as - select * from connection to &lib( - select - s.name as SchemaName, - t.name as memname, - tc.name as name, - ic.key_ordinal as KeyOrderNr - from - sys.schemas s - inner join sys.tables t on s.schema_id=t.schema_id - inner join sys.indexes i on t.object_id=i.object_id - inner join sys.index_columns ic on i.object_id=ic.object_id - and i.index_id=ic.index_id - inner join sys.columns tc on ic.object_id=tc.object_id - and ic.column_id=tc.column_id - where i.is_primary_key=1 - and s.name=%str(%')&schema%str(%') - order by t.name, ic.key_ordinal ; - );disconnect from &lib; - create table &ds3 as - select a.* - ,case when b.name is not null then 1 else 0 end as pk_ind - from work.&ds1 a - left join work.&ds2 b - on a.dsn=b.memname - and upcase(a.name)=upcase(b.name) - order by libref,dsn; -%end; -%else %do; - - %if &ds = 0 %then %let dsn=; - - /* get all constraints, in constraint order*/ - %mp_getconstraints(lib=&lib,ds=&dsn,outds=work.&ds2) - - /* extract cols that are clearly primary keys */ - proc sql; - create table &pk4sure as - select libref - ,table_name - ,constraint_name - ,constraint_order - ,column_name as name - from work.&ds2 - where constraint_type='PRIMARY' - order by 1,2,3,4; - - /* extract unique constraints where every col is also NOT NULL */ - proc sql; - create table &sum as - select a.libref - ,a.table_name - ,a.constraint_name - ,count(a.column_name) as unq_cnt - ,count(b.column_name) as nul_cnt - from work.&ds2(where=(constraint_type ='UNIQUE')) a - left join work.&ds2(where=(constraint_type ='NOT NULL')) b - on a.libref=b.libref - and a.table_name=b.table_name - and a.column_name=b.column_name - group by 1,2,3 - having unq_cnt=nul_cnt; - - /* extract cols from the relevant unique constraints */ - create table &pkdefault as - select a.libref - ,a.table_name - ,a.constraint_name - ,b.constraint_order - ,b.column_name as name - from &sum a - left join &ds2(where=(constraint_type ='UNIQUE')) b - on a.libref=b.libref - and a.table_name=b.table_name - and a.constraint_name=b.constraint_name - order by 1,2,3,4; - - /* extract cols from the relevant unique INDEXES */ - create table &pkfromindex as - select libname as libref - ,memname as table_name - ,indxname as constraint_name - ,indxpos as constraint_order - ,name - from dictionary.indexes - where nomiss='yes' and unique='yes' and upcase(libname)="&lib" - %if &ds ne 0 %then %do; - and upcase(memname)="&ds" - %end; - order by 1,2,3,4; - - /* create one table */ - data &finalpks; - set &pkdefault &pk4sure &pkfromindex; - pk_ind=1; - /* if there are multiple unique constraints, take the first */ - by libref table_name constraint_name; - retain keepme; - if first.table_name then keepme=1; - if first.constraint_name and not first.table_name then keepme=0; - if keepme=1; - run; - - /* join back to starting table */ - proc sql; - create table &ds3 as - select a.* - ,b.constraint_order - ,case when b.pk_ind=1 then 1 else 0 end as pk_ind - from work.&ds1 a - left join work.&finalpks b - on a.libref=b.libref - and a.dsn=b.table_name - and upcase(a.name)=upcase(b.name) - order by libref,dsn,constraint_order; -%end; - - -/* prepare tables */ -proc sql; -create table work.&tabs1 as select - libname as libref - ,upcase(memname) as dsn - ,memtype -%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do; - ,dbms_memtype -%end; -%else %do; - ,'n/a' as dbms_memtype format=$32. -%end; - ,typemem - ,memlabel - ,nvar - ,compress -from dictionary.tables - where upcase(libname)="&lib" -%if &ds ne 0 %then %do; - and upcase(memname)="&ds" -%end; - ; -data &tabs2; - set &ds3; - length pk_fields $512; - retain pk_fields; - by libref dsn constraint_order; - if first.dsn then pk_fields=''; - if pk_ind=1 then pk_fields=catx(' ',pk_fields,name); - if last.dsn then output; -run; - -proc sql; -create table &outds as - select a.libref - ,a.dsn - ,a.memtype - ,a.dbms_memtype - ,a.typemem - ,a.memlabel - ,a.nvar - ,a.compress - ,b.pk_fields - from work.&tabs1 a - left join work.&tabs2 b - on a.libref=b.libref - and a.dsn=b.dsn; - -/* tidy up */ -%mp_dropmembers( - &ds1 &ds2 &ds3 &dsn &tabs1 &tabs2 &sum &pk4sure &pkdefault &finalpks, - iftrue=(&mdebug=0) -) - -%mend mp_getpk; diff --git a/009_macros/mp_gitadd.sas b/009_macros/mp_gitadd.sas deleted file mode 100644 index dcece31..0000000 --- a/009_macros/mp_gitadd.sas +++ /dev/null @@ -1,45 +0,0 @@ -/*** HELP START ***//** - @file - @brief Stages files in a GIT repo - @details Uses the output dataset from mp_gitstatus.sas to determine the files - that should be staged. - - If `STAGED ne "TRUE"` then the file is staged. - - Usage: - - %let dir=%sysfunc(pathname(work))/core; - %let repo=https://github.com/sasjs/core; - %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir)); - %mf_writefile(&dir/somefile.txt,l1=some content) - %mf_deletefile(&dir/package.json) - %mp_gitstatus(&dir,outds=work.gitstatus) - - %mp_gitadd(&dir,inds=work.gitstatus) - - @param [in] gitdir The directory containing the GIT repository - @param [in] inds= (work.mp_gitadd) The input dataset with the list of files - to stage. Will accept the output from mp_gitstatus(), else just use a table - with the following columns: - @li path $1024 - relative path to the file in the repo - @li staged $32 - whether the file is staged (TRUE or FALSE) - @li status $64 - either new, deleted, or modified - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

Related Files

- @li mp_gitadd.test.sas - @li mp_gitstatus.sas - -**//*** HELP END ***/ - -%macro mp_gitadd(gitdir,inds=work.mp_gitadd,mdebug=0); - -data _null_; - set &inds; - if STAGED ne "TRUE"; - rc=git_index_add("&gitdir",cats(path),status); - if rc ne 0 or &mdebug=1 then put rc=; -run; - -%mend mp_gitadd; diff --git a/009_macros/mp_gitlog.sas b/009_macros/mp_gitlog.sas deleted file mode 100644 index 71d04c4..0000000 --- a/009_macros/mp_gitlog.sas +++ /dev/null @@ -1,104 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with the commit history of a local repository - @details Returns the commit history from a local repository. The name of the - branch is also returned. - - More details here: -https://documentation.sas.com/doc/ko/pgmsascdc/v_033/lefunctionsref/n1qo5miyvry1nen111js203hlwrh.htm - - Usage: - - %let gitdir=%sysfunc(pathname(work))/core; - %let repo=https://github.com/sasjs/core; - %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir)); - - %mp_gitlog(&gitdir,outds=work.mp_gitlog) - - @param [in] gitdir The directory containing the GIT repository - @param [in] filter= (BRANCHONLY) To return only the commits for the current - branch, use BRANCHONLY (the default). Anything else will return the entire - commit history. - @param [out] outds= (work.mp_gitlog) The output dataset to create. - All vars are $128 except `message` which is $4000. - @li author returns the author who submitted the commit. - @li children_ids returns a list of the children commit IDs - @li committer returns the name of the committer. - @li committer_email returns the email of the committer. - @li email returns the email of the commit author. - @li id returns the commit ID of the commit object. - @li in_current_branch returns "TRUE" or "FALSE" to indicate if the commit is - in the current branch. - @li message returns the commit message. - @li parent_ids returns a list of the parent commit IDs. - @li stash returns "TRUE" or "FALSE" to indicate if the commit is a stash - commit. - @li time returns the time of the commit as numeric string - @li commit_time_num time of the commit as numeric SAS datetime - @li commit_time_str the commit_time_num variable cast as string - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [in] nobs= (0) Set to an integer greater than 0 to restrict the number - of rows returned - -

SAS Macros

- @li mf_getgitbranch.sas - -

Related Files

- @li mp_gitadd.sas - @li mp_gitreleaseinfo.sas - @li mp_gitstatus.sas - -**//*** HELP END ***/ - -%macro mp_gitlog(gitdir,outds=work.mp_gitlog,mdebug=0,filter=BRANCHONLY,nobs=0); - -%local varlist i var; -%let varlist=author children_ids committer committer_email email id - in_current_branch parent_ids stash time ; - -data &outds; - LENGTH gitdir branch $ 1024 message $4000 &varlist $128 commit_time_num 8. - commit_time_str $32; - call missing (of _all_); - branch="%mf_getgitbranch(&gitdir)"; - gitdir=symget('gitdir'); - rc=git_status_free(trim(gitdir)); - if rc=-1 then do; - put "The libgit2 library is unavailable and no Git operations can be used."; - put "See: https://stackoverflow.com/questions/74082874"; - stop; - end; - else if rc=-2 then do; - put "The libgit2 library is available, but the status function failed."; - put "See the log for details."; - stop; - end; - entries=git_commit_log(trim(gitdir)); - do n=1 to entries; - - %do i=1 %to %sysfunc(countw(&varlist message)); - %let var=%scan(&varlist message,&i,%str( )); - rc=git_commit_get(n,trim(gitdir),"&var",&var); - %end; - /* convert unix time to SAS time - https://4gl.uk/corelink0 */ - /* Number of seconds between 01JAN1960 and 01JAN1970: 315619200 */ - format commit_time_num datetime19.; - commit_time_num=sum(input(cats(time),best.),315619200); - commit_time_str=put(commit_time_num,datetime19.); - %if &mdebug=1 %then %do; - putlog (_all_)(=); - %end; - if "&filter"="BRANCHONLY" then do; - if cats(in_current_branch)='TRUE' then output; - end; - else output; - %if &nobs>0 %then %do; - if n ge &nobs then stop; - %end; - end; - rc=git_commit_free(trim(gitdir)); - keep gitdir branch &varlist message time commit_time_num commit_time_str; -run; - -%mend mp_gitlog; diff --git a/009_macros/mp_gitreleaseinfo.sas b/009_macros/mp_gitreleaseinfo.sas deleted file mode 100644 index 006c977..0000000 --- a/009_macros/mp_gitreleaseinfo.sas +++ /dev/null @@ -1,74 +0,0 @@ -/*** HELP START ***//** - @file - @brief Pulls latest release info from a GIT repository - @details Useful for grabbing the latest version number or other attributes - from a GIT server. Supported providers are GitLab and GitHub. Pull requests - are welcome if you'd like to see additional providers! - - Note that each provider provides slightly different JSON output. Therefore - the macro simply extracts the JSON and assigns the libname (using the JSON - engine). - - Example usage (eg, to grab latest release version from github): - - %mp_gitreleaseinfo(GITHUB,sasjs/core,outlib=mylibref) - - data _null_; - set mylibref.root; - putlog TAG_NAME=; - run; - - @param [in] provider The GIT provider for the release info. Accepted values: - @li GITLAB - @li GITHUB - Tables include root, assets, author, alldata - @param [in] project The link to the repository. This has different formats - depending on the vendor: - @li GITHUB - org/repo, eg sasjs/core - @li GITLAB - project, eg 1343223 - @param [in] server= (0) If your repo is self-hosted, then provide the domain - here. Otherwise it will default to the provider domain (eg gitlab.com). - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - @param [out] outlib= (GITREL) The JSON-engine libref to be created, which will - point at the returned JSON - -

SAS Macros

- @li mf_getuniquefileref.sas - -

Related Files

- @li mp_gitreleaseinfo.test.sas - -**//*** HELP END ***/ - -%macro mp_gitreleaseinfo(provider,project,server=0,outlib=GITREL,mdebug=0); -%local url fref; - -%let provider=%upcase(&provider); - -%if &provider=GITHUB %then %do; - %if "&server"="0" %then %let server=https://api.github.com; - %let url=&server/repos/&project/releases/latest; -%end; -%else %if &provider=GITLAB %then %do; - %if "&server"="0" %then %let server=https://gitlab.com; - %let url=&server/api/v4/projects/&project/releases; -%end; - -%let fref=%mf_getuniquefileref(); - -proc http method='GET' out=&fref url="&url"; -%if &mdebug=1 %then %do; - debug level = 3; -%end; -run; - -libname &outlib JSON fileref=&fref; - -%if &mdebug=1 %then %do; - data _null_; - infile &fref; - input; - putlog _infile_; - run; -%end; - -%mend mp_gitreleaseinfo; diff --git a/009_macros/mp_gitstatus.sas b/009_macros/mp_gitstatus.sas deleted file mode 100644 index 758810d..0000000 --- a/009_macros/mp_gitstatus.sas +++ /dev/null @@ -1,67 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a dataset with the output from `GIT_STATUS()` - @details Uses `git_status()` to fetch the number of changed files, then - iterates with `git_status_get()`, inserting all attributes into an output - dataset. - - Usage: - - %let dir=%sysfunc(pathname(work))/core; - %let repo=https://github.com/sasjs/core; - %put source clone rc=%sysfunc(GITFN_CLONE(&repo,&dir)); - %mf_writefile(&dir/somefile.txt,l1=some content) - %mf_deletefile(&dir/package.json) - - %mp_gitstatus(&dir,outds=work.gitstatus) - - More info on these functions is in this [helpful paper] -(https://www.sas.com/content/dam/SAS/support/en/sas-global-forum-proceedings/2019/3057-2019.pdf) - by Danny Zimmerman. - - @param [in] gitdir The directory containing the GIT repository - @param [out] outds= (work.git_status) The output dataset to create. Vars: - @li gitdir $1024 - directory of repo - @li path $1024 - relative path to the file in the repo - @li staged $32 - whether the file is staged (TRUE or FALSE) - @li status $64 - either new, deleted, or modified - @li cnt - number of files - @li n - the "nth" file in the list from git_status() - - @param [in] mdebug= (0) Set to 1 to enable DEBUG messages - -

Related Files

- @li mp_gitstatus.test.sas - @li mp_gitadd.sas - -**//*** HELP END ***/ - -%macro mp_gitstatus(gitdir,outds=work.mp_gitstatus,mdebug=0); - -data &outds; - LENGTH gitdir path $ 1024 STATUS $ 64 STAGED $ 32; - call missing (of _all_); - gitdir=symget('gitdir'); - cnt=git_status(trim(gitdir)); - if cnt=-1 then do; - put "The libgit2 library is unavailable and no Git operations can be used."; - put "See: https://stackoverflow.com/questions/74082874"; - end; - else if cnt=-2 then do; - put "The libgit2 library is available, but the status function failed."; - put "See the log for details."; - end; - else do n=1 to cnt; - rc=GIT_STATUS_GET(n,gitdir,'PATH',path); - rc=GIT_STATUS_GET(n,gitdir,'STAGED',staged); - rc=GIT_STATUS_GET(n,gitdir,'STATUS',status); - output; - %if &mdebug=1 %then %do; - putlog (_all_)(=); - %end; - end; - rc=git_status_free(trim(gitdir)); - drop rc cnt; -run; - -%mend mp_gitstatus; diff --git a/009_macros/mp_gsubfile.sas b/009_macros/mp_gsubfile.sas deleted file mode 100644 index fa62655..0000000 --- a/009_macros/mp_gsubfile.sas +++ /dev/null @@ -1,58 +0,0 @@ -/*** HELP START ***//** - @file - @brief Performs a text substitution on a file - @details Makes use of the GSUB function in LUA to perform a text substitution - in a file - either in-place, or writing to a new location. The benefit of - using LUA is that the entire file can be loaded into a single variable, - thereby side stepping the 32767 character limit in a data step. - - Usage: - - %let file=%sysfunc(pathname(work))/file.txt; - %let str=replace/me; - %let rep=with/this; - data _null_; - file "&file"; - put "&str"; - run; - %mp_gsubfile(file=&file, patternvar=str, replacevar=rep) - data _null_; - infile "&file"; - input; - list; - run; - - @param [in] file= (0) The file to perform the substitution on - @param [in] patternvar= A macro variable containing the Lua - [pattern](https://www.lua.org/pil/20.2.html) to search for. Due to the use - of special (magic) characters in Lua patterns, it is safer to pass the NAME - of the macro variable containing the string, rather than the value itself. - @param [in] replacevar= () - The name of the macro variable containing the replacement _string_. - @param [out] outfile= (0) The file to write the output to. - If zero, then the file is overwritten in-place. - -

SAS Macros

- @li ml_gsubfile.sas - -

Related Macros

- @li mp_gsubfile.test.sas - - @version 9.4 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_gsubfile(file=0, - patternvar=, - replacevar=, - outfile=0 -)/*/STORE SOURCE*/; - - %if "%substr(&sysver.XX,1,4)"="V.04" %then %do; - %put %str(ERR)OR: Viya 4 does not support the IO library in lua; - %return; - %end; - - %ml_gsubfile() - -%mend mp_gsubfile; diff --git a/009_macros/mp_guesspk.sas b/009_macros/mp_guesspk.sas deleted file mode 100644 index 2106014..0000000 --- a/009_macros/mp_guesspk.sas +++ /dev/null @@ -1,332 +0,0 @@ -/*** HELP START ***//** - @file - @brief Guess the primary key of a table - @details Tries to guess the primary key of a table based on the following - logic: - - * Columns with nulls are ignored - * Return only column combinations that provide unique results - * Start from one column, then move out to composite keys of 2 to 6 columns - - The library of the target should be assigned before using this macro. - - Usage: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - %mp_guesspk(sashelp.class,outds=classpks) - - @param [in] baseds The dataset to analyse - @param [out] outds= (mp_guesspk) Output dataset to contain the possible PKs - @param [in] max_guesses= (3) The total number of possible primary keys to - generate. A table may have multiple (unlikely) PKs, so no need to list them - all. - @param [in] min_rows= (5) The minimum number of rows a table should have in - order to try and guess the PK. - @param [in] ignore_cols (0) Space seperated list of columns which you are - sure are not part of the primary key (helps to avoid false positives) - @param [in] mdebug= Set to 1 to enable DEBUG messages and preserve outputs - -

SAS Macros

- @li mf_getvarlist.sas - @li mf_getuniquename.sas - @li mf_wordsInstr1butnotstr2.sas - @li mf_nobs.sas - -

Related Macros

- @li mp_getpk.sas - - @version 9.3 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_guesspk(baseds - ,outds=mp_guesspk - ,max_guesses=3 - ,min_rows=5 - ,ignore_cols=0 - ,mdebug=0 -)/*/STORE SOURCE*/; -%local dbg; -%if &mdebug=1 %then %do; - %put &sysmacroname entry vars:; - %put _local_; -%end; -%else %let dbg=*; - -/* declare local vars */ -%local var vars vcnt i j k l tmpvar tmpds rows posspks ppkcnt; -%let vars=%upcase(%mf_getvarlist(&baseds)); -%let vars=%mf_wordsInStr1ButNotStr2(str1=&vars,str2=%upcase(&ignore_cols)); -%let vcnt=%sysfunc(countw(&vars)); - -%if &vcnt=0 %then %do; - %put &sysmacroname: &baseds has no variables! Exiting.; - %return; -%end; - -/* get null count and row count */ -%let tmpvar=%mf_getuniquename(); -proc sql noprint; -create table _data_ as select - count(*) as &tmpvar -%do i=1 %to &vcnt; - %let var=%scan(&vars,&i); - ,sum(case when &var is missing then 1 else 0 end) as &var -%end; - from &baseds; - -/* transpose table and scan for not null cols */ -proc transpose; -data _null_; - set &syslast end=last; - length vars $32767; - retain vars ; - if _name_="&tmpvar" then call symputx('rows',col1,'l'); - else if col1=0 then vars=catx(' ',vars,_name_); - if last then call symputx('posspks',vars,'l'); -run; - -%let ppkcnt=%sysfunc(countw(&posspks)); -%if &ppkcnt=0 %then %do; - %put &sysmacroname: &baseds has no non-missing variables! Exiting.; - %return; -%end; - -proc sort data=&baseds(keep=&posspks) out=_data_ noduprec; - by _all_; -run; -%local pkds; %let pkds=&syslast; - -%if &rows > %mf_nobs(&pkds) %then %do; - %put &sysmacroname: &baseds has no combination of unique records! Exiting.; - %return; -%end; - -/* now check cardinality */ -proc sql noprint; -create table _data_ as select -%do i=1 %to &ppkcnt; - %let var=%scan(&posspks,&i); - count(distinct &var) as &var - %if &i<&ppkcnt %then ,; -%end; - from &pkds; - -/* transpose and sort by cardinality */ -proc transpose; -proc sort; by descending col1; -run; - -/* create initial PK list and re-order posspks list */ -data &outds(keep=pkguesses); - length pkguesses $5000 vars $5000; - set &syslast end=last; - retain vars ; - vars=catx(' ',vars,_name_); - if col1=&rows then do; - pkguesses=_name_; - output; - end; - if last then call symputx('posspks',vars,'l'); -run; - -%if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: %mf_nobs(&outds) possible primary key values found; - %return; -%end; - -%if &ppkcnt=1 %then %do; - %put &sysmacroname: No more PK guess possible; - %return; -%end; - -/* begin scanning for uniques on pairs of PKs */ -%let tmpds=%mf_getuniquename(); -%local lev1 lev2; -%do i=1 %to &ppkcnt; - %let lev1=%scan(&posspks,&i); - %do j=2 %to &ppkcnt; - %let lev2=%scan(&posspks,&j); - %if &lev1 ne &lev2 %then %do; - /* check for two level uniqueness */ - proc sort data=&pkds(keep=&lev1 &lev2) out=&tmpds noduprec; - by _all_; - run; - %if %mf_nobs(&tmpds)=&rows %then %do; - proc sql; - insert into &outds values("&lev1 &lev2"); - %if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: Max PKs reached at Level 2 for &baseds; - %goto exit; - %end; - %end; - %end; - %end; -%end; - -%if &ppkcnt=2 %then %do; - %put &sysmacroname: No more PK guess possible; - %goto exit; -%end; - -/* begin scanning for uniques on PK triplets */ -%local lev3; -%do i=1 %to &ppkcnt; - %let lev1=%scan(&posspks,&i); - %do j=2 %to &ppkcnt; - %let lev2=%scan(&posspks,&j); - %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt; - %let lev3=%scan(&posspks,&k); - %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do; - /* check for three level uniqueness */ - proc sort data=&pkds(keep=&lev1 &lev2 &lev3) out=&tmpds noduprec; - by _all_; - run; - %if %mf_nobs(&tmpds)=&rows %then %do; - proc sql; - insert into &outds values("&lev1 &lev2 &lev3"); - %if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: Max PKs reached at Level 3 for &baseds; - %goto exit; - %end; - %end; - %end; - %end; - %end; -%end; - -%if &ppkcnt=3 %then %do; - %put &sysmacroname: No more PK guess possible; - %goto exit; -%end; - -/* scan for uniques on up to 4 PK fields */ -%local lev4; -%do i=1 %to &ppkcnt; - %let lev1=%scan(&posspks,&i); - %do j=2 %to &ppkcnt; - %let lev2=%scan(&posspks,&j); - %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt; - %let lev3=%scan(&posspks,&k); - %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt; - %let lev4=%scan(&posspks,&l); - %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then %do; - /* check for four level uniqueness */ - proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4) - out=&tmpds noduprec; - by _all_; - run; - %if %mf_nobs(&tmpds)=&rows %then %do; - proc sql; - insert into &outds values("&lev1 &lev2 &lev3 &lev4"); - %if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: Max PKs reached at Level 4 for &baseds; - %goto exit; - %end; - %end; - %end; - %end; - %end; - %end; -%end; - -%if &ppkcnt=4 %then %do; - %put &sysmacroname: No more PK guess possible; - %goto exit; -%end; - -/* scan for uniques on up to 4 PK fields */ -%local lev5 m; -%do i=1 %to &ppkcnt; - %let lev1=%scan(&posspks,&i); - %do j=2 %to &ppkcnt; - %let lev2=%scan(&posspks,&j); - %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt; - %let lev3=%scan(&posspks,&k); - %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt; - %let lev4=%scan(&posspks,&l); - %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then - %do m=5 %to &ppkcnt; - %let lev5=%scan(&posspks,&m); - %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 %then %do; - /* check for four level uniqueness */ - proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5) - out=&tmpds noduprec; - by _all_; - run; - %if %mf_nobs(&tmpds)=&rows %then %do; - proc sql; - insert into &outds values("&lev1 &lev2 &lev3 &lev4 &lev5"); - %if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: Max PKs reached at Level 5 for &baseds; - %goto exit; - %end; - %end; - %end; - %end; - %end; - %end; - %end; -%end; - -%if &ppkcnt=5 %then %do; - %put &sysmacroname: No more PK guess possible; - %goto exit; -%end; - -/* scan for uniques on up to 4 PK fields */ -%local lev6 n; -%do i=1 %to &ppkcnt; - %let lev1=%scan(&posspks,&i); - %do j=2 %to &ppkcnt; - %let lev2=%scan(&posspks,&j); - %if &lev1 ne &lev2 %then %do k=3 %to &ppkcnt; - %let lev3=%scan(&posspks,&k); - %if &lev1 ne &lev3 and &lev2 ne &lev3 %then %do l=4 %to &ppkcnt; - %let lev4=%scan(&posspks,&l); - %if &lev1 ne &lev4 and &lev2 ne &lev4 and &lev3 ne &lev4 %then - %do m=5 %to &ppkcnt; - %let lev5=%scan(&posspks,&m); - %if &lev1 ne &lev5 & &lev2 ne &lev5 & &lev3 ne &lev5 & &lev4 ne &lev5 - %then %do n=6 %to &ppkcnt; - %let lev6=%scan(&posspks,&n); - %if &lev1 ne &lev6 & &lev2 ne &lev6 & &lev3 ne &lev6 - & &lev4 ne &lev6 & &lev5 ne &lev6 %then - %do; - /* check for four level uniqueness */ - proc sort data=&pkds(keep=&lev1 &lev2 &lev3 &lev4 &lev5 &lev6) - out=&tmpds noduprec; - by _all_; - run; - %if %mf_nobs(&tmpds)=&rows %then %do; - proc sql; - insert into &outds - values("&lev1 &lev2 &lev3 &lev4 &lev5 &lev6"); - %if %mf_nobs(&outds) ge &max_guesses %then %do; - %put &sysmacroname: Max PKs reached at Level 6 for &baseds; - %goto exit; - %end; - %end; - %end; - %end; - %end; - %end; - %end; - %end; -%end; - -%if &ppkcnt=6 %then %do; - %put &sysmacroname: No more PK guess possible; - %goto exit; -%end; - -%exit: -%if &mdebug=0 %then %do; - proc sql; - drop table &tmpds; -%end; - -%mend mp_guesspk; diff --git a/009_macros/mp_hashdataset.sas b/009_macros/mp_hashdataset.sas deleted file mode 100644 index 3005dad..0000000 --- a/009_macros/mp_hashdataset.sas +++ /dev/null @@ -1,91 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a unique hash for a dataset - @details Ignores metadata attributes, used only to hash values. If used to - compare datasets, they must have their columns and rows in the same order. - - %mp_hashdataset(sashelp.class,outds=myhash) - - data _null_; - set work.myhash; - put hashkey=; - run; - - ![sas md5 hash dataset log results](https://i.4gl.io/1/KorUKoyE05.png/raw) - -

SAS Macros

- @li mf_getattrn.sas - @li mf_getuniquename.sas - @li mf_getvarlist.sas - @li mp_md5.sas - -

Related Files

- @li mp_hashdataset.test.sas - @li mp_hashdirectory.sas - - @param [in] libds dataset to hash - @param [in] salt= () Provide a salt (could be, for instance, the dataset name) - @param [in] iftrue= (1=1) A condition under which the macro should be executed - @param [out] outds= (work._data_) The output dataset to create. This - will contain one column (hashkey) with one observation (a $hex32. - representation of the input hash) - |hashkey:$32.| - |---| - |28ABC74ABFC45F50794237BA5566E6CA| - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_hashdataset( - libds, - outds=work._data_, - salt=, - iftrue=%str(1=1) -)/*/STORE SOURCE*/; - -%local keyvar /* roll up the md5 */ - prevkeyvar /* retain prev record md5 */ - lastvar /* last var in input ds */ - cvars nvars; - -%if not(%eval(%unquote(&iftrue))) %then %return; - -/* avoid naming conflict for hash key vars */ -%let keyvar=%mf_getuniquename(); -%let prevkeyvar=%mf_getuniquename(); -%let lastvar=%mf_getuniquename(); - -%if %mf_getattrn(&libds,NLOBS)=0 %then %do; - data &outds; - length hashkey $32; - hashkey=put(md5("&salt"),$hex32.); - output; - stop; - run; - %put &sysmacroname: Dataset &libds is empty, or is not a dataset; - %put &sysmacroname: hashkey of &outds is based on salt (&salt) only; -%end; -%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do; - %put %str(ERR)OR: Dataset &libds is not a dataset; -%end; -%else %do; - data &outds(rename=(&keyvar=hashkey) keep=&keyvar) - %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - /nonote2err - %end; - ; - length &prevkeyvar &keyvar $32; - retain &prevkeyvar; - if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.); - set &libds end=&lastvar; - /* hash should include previous row */ - &keyvar=%mp_md5( - cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar, - nvars=%mf_getvarlist(&libds,typefilter=N) - ); - &prevkeyvar=&keyvar; - if &lastvar then output; - run; -%end; -%mend mp_hashdataset; diff --git a/009_macros/mp_hashdirectory.sas b/009_macros/mp_hashdirectory.sas deleted file mode 100644 index 36a3072..0000000 --- a/009_macros/mp_hashdirectory.sas +++ /dev/null @@ -1,164 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns a unique hash for each file in a directory - @details Hashes each file in each directory, and then hashes the hashes to - create a hash for each directory also. - - This makes use of the new `hashing_file()` and `hashing` functions, available - since 9.4m6. Interestingly, those functions can be used in pure macro, eg: - - %put %sysfunc(hashing_file(md5,/path/to/file.blob,0)); - - Actual usage: - - %let fpath=/some/directory; - - %mp_hashdirectory(&fpath,outds=myhash,maxdepth=2) - - data _null_; - set work.myhash; - put (_all_)(=); - run; - - Whilst files are hashed in their entirety, the logic for creating a folder - hash is as follows: - - @li Sort the files by filename (case sensitive, uppercase then lower) - @li Take the first 100 hashes, concatenate and hash - @li Concatenate this hash with another 100 hashes and hash again - @li Continue until the end of the folder. This is the folder hash - @li If a folder contains other folders, start from the bottom of the tree - - the folder hashes cascade upwards so you know immediately if there is a - change in a sub/sub directory - @li If a subfolder has no content (empty) then it is ignored. No hash created. - @li If the file is empty, it is also ignored / no hash created. - @li If the target directory (&inloc) is empty, &outds will also be empty - -

SAS Macros

- @li mp_dirlist.sas - -

Related Files

- @li mp_hashdataset.sas - @li mp_hashdirectory.test.sas - @li mp_md5.sas - - @param [in] inloc Full filepath of the file to be hashed (unquoted) - @param [in] iftrue= (1=1) A condition under which the macro should be executed - @param [in] maxdepth= (0) Set to a positive integer to indicate the level of - subdirectory scan recursion - eg 3, to go `./3/levels/deep`. For unlimited - recursion, set to MAX. - @param [in] method= (MD5) the hashing method to use. Available options: - @li MD5 - @li SH1 - @li SHA256 - @li SHA384 - @li SHA512 - @li CRC32 - @param [out] outds= (work.mp_hashdirectory) The output dataset. Contains: - @li directory - the parent folder - @li file_hash - the hash output - @li hash_duration - how long the hash took (first hash always takes longer) - @li file_path - /full/path/to/each/file.ext - @li file_or_folder - contains either "file" or "folder" - @li level - the depth of the directory (top level is 0) - - @version 9.4m6 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_hashdirectory(inloc, - outds=work.mp_hashdirectory, - method=MD5, - maxdepth=0, - iftrue=%str(1=1) -)/*/STORE SOURCE*/; - -%local curlevel tempds maxlevel; - -%if not(%eval(%unquote(&iftrue))) %then %return; - -/* get the directory listing */ -%mp_dirlist(path=&inloc, outds=&outds, maxdepth=&maxdepth, showparent=YES) - -/* create the hashes */ -data &outds; - set &outds (rename=(filepath=file_path)); - length FILE_HASH $32 HASH_DURATION 8; - keep directory file_hash hash_duration file_path file_or_folder level; - - ts=datetime(); - if file_or_folder='file' then do; - /* if file is empty, hashing_file will break - so ignore / delete */ - length fname val $8; - drop fname val fid is_empty; - rc=filename(fname,file_path); - fid=fopen(fname); - if fid > 0 then do; - rc=fread(fid); - is_empty=fget(fid,val); - end; - rc=fclose(fid); - rc=filename(fname); - if is_empty ne 0 then delete; - else file_hash=hashing_file("&method",cats(file_path),0); - end; - hash_duration=datetime()-ts; -run; - -proc sort data=&outds ; - by descending level directory file_path; -run; - -%let maxlevel=0; -data _null_; - set &outds; - call symputx('maxlevel',level,'l'); - stop; -run; - -/* now hash the hashes to populate folder hashes, starting from the bottom */ -%do curlevel=&maxlevel %to 0 %by -1; - data work._data_ (keep=directory file_hash); - set &outds; - where level=&curlevel; - by descending level directory file_path; - length str $32767 tmp_hash $32; - retain str tmp_hash ; - /* reset vars when starting a new directory */ - if first.directory then do; - str=''; - tmp_hash=''; - i=0; - end; - /* hash each chunk of 100 file paths */ - i+1; - str=cats(str,file_hash); - if mod(i,100)=0 or last.directory then do; - tmp_hash=hashing("&method",cats(tmp_hash,str)); - str=''; - end; - /* output the hash at directory level */ - if last.directory then do; - file_hash=tmp_hash; - output; - end; - if last.level then stop; - run; - %let tempds=&syslast; - /* join the hash back into the main table */ - proc sql undo_policy=none; - create table &outds as - select a.directory - ,coalesce(b.file_hash,a.file_hash) as file_hash - ,a.hash_duration - ,a.file_path - ,a.file_or_folder - ,a.level - from &outds a - left join &tempds b - on a.file_path=b.directory - order by level desc, directory, file_path; - drop table &tempds; -%end; - -%mend mp_hashdirectory; diff --git a/009_macros/mp_include.sas b/009_macros/mp_include.sas deleted file mode 100644 index 57386d8..0000000 --- a/009_macros/mp_include.sas +++ /dev/null @@ -1,106 +0,0 @@ -/*** HELP START ***//** - @file - @brief Performs a wrapped \%include - @details This macro wrapper is necessary if you need your included code to - know that it is being \%included. - - If you are using %include in a regular program, you could make use of the - following macro variables: - - @li SYSINCLUDEFILEDEVICE - @li SYSINCLUDEFILEDIR - @li SYSINCLUDEFILEFILEREF - @li SYSINCLUDEFILENAME - - However these variables are NOT available inside a macro, as documented here: -https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/mcrolref/n1j5tcc0n2xczyn1kg1o0606gsv9.htm - - This macro can be used in place of the %include statement, and will insert - the following (equivalent) global variables: - - @li _SYSINCLUDEFILEDEVICE - @li _SYSINCLUDEFILEDIR - @li _SYSINCLUDEFILEFILEREF - @li _SYSINCLUDEFILENAME - - These can be used whenever testing _within a macro_. Outside of the macro, - the regular automatic variables will still be available (thanks to a - concatenated file list in the include statement). - - Example usage: - - filename example temp; - data _null_; - file example; - put '%macro test();'; - put '%put &=_SYSINCLUDEFILEFILEREF;'; - put '%put &=SYSINCLUDEFILEFILEREF;'; - put '%mend; %test()'; - put '%put &=SYSINCLUDEFILEFILEREF;'; - run; - %mp_include(example) - - @param [in] fileref The fileref of the file to be included. Must be provided. - @param [in] prefix= (_) The prefix to apply to the global variables. - @param [in] opts= (SOURCE2) The options to apply to the %inc statement - @param [in] errds= (work.mp_abort_errds) There is no clean way to end a - process within a %include called within a macro. Furthermore, there is no - way to test if a macro is called within a %include. To handle this - particular scenario, the %mp_abort() macro will test for the existence of - the `_SYSINCLUDEFILEDEVICE` variable and return the outputs (msg,mac) inside - this dataset. - It will then run an abort cancel FILE to stop the include running, and pass - the dataset back. - - IMPORTANT NOTE - it is NOT possible to read this dataset as part of _this_ - macro! When running abort cancel FILE, ALL macros are closed, so instead it - is necessary to invoke "%mp_abort(mode=INCLUDE)" OUTSIDE of macro wrappers. - - - @version 9.4 - @author Allan Bowe - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mp_abort.sas - -**//*** HELP END ***/ - -%macro mp_include(fileref - ,prefix=_ - ,opts=SOURCE2 - ,errds=work.mp_abort_errds -)/*/STORE SOURCE*/; - -/* prepare precode */ -%local tempref; -%let tempref=%mf_getuniquefileref(); -data _null_; - file &tempref; - set sashelp.vextfl(where=(fileref="%upcase(&fileref)")); - put '%let _SYSINCLUDEFILEDEVICE=' xengine ';'; - name=scan(xpath,-1,'/\'); - put '%let _SYSINCLUDEFILENAME=' name ';'; - path=subpad(xpath,1,length(xpath)-length(name)-1); - put '%let _SYSINCLUDEFILEDIR=' path ';'; - put '%let _SYSINCLUDEFILEFILEREF=' "&fileref;"; -run; - -/* prepare the errds */ -data &errds; - length msg mac $1000; - call missing(msg,mac); - iftrue='1=0'; -run; - -/* include the include */ -%inc &tempref &fileref/&opts; - -%mp_abort(iftrue= (&syscc ne 0) - ,mac=%str(&_SYSINCLUDEFILEDIR/&_SYSINCLUDEFILENAME) - ,msg=%str(syscc=&syscc after executing &_SYSINCLUDEFILENAME) -) - -filename &tempref clear; - -%mend mp_include; diff --git a/009_macros/mp_init.sas b/009_macros/mp_init.sas deleted file mode 100644 index 78d82ff..0000000 --- a/009_macros/mp_init.sas +++ /dev/null @@ -1,76 +0,0 @@ -/*** HELP START ***//** - @file - @brief Initialise session with useful settings and variables - @details Implements a "strict" set of SAS options for use in defensive - programming. Highly recommended, if you want your code to run on some - other machine. - - This macro is recommended to be compiled and invoked in the `initProgram` - for SASjs [Jobs](https://cli.sasjs.io/sasjsconfig.html#jobConfig_initProgram - ), [Services]( - https://cli.sasjs.io/sasjsconfig.html#serviceConfig_initProgram) and [Tests] - (https://cli.sasjs.io/sasjsconfig.html#testConfig_initProgram). - - For non SASjs projects, you could invoke in the autoexec, or in your own - solution initialisation macro. - - - If you have a good idea for another useful option, setting, or global - variable - feel free to [raise an issue]( - https://github.com/sasjs/core/issues/new)! - - All global variables are prefixed with "SASJS" (unless modified with the - prefix parameter). - - @param [in] prefix= (SASJS) The prefix to apply to the global macro variables - - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_init(prefix=SASJS -)/*/STORE SOURCE*/; - -%if %symexist(SASJS_PREFIX) %then %return; /* only run once */ - -%global - SASJS_PREFIX /* the ONLY hard-coded global macro variable in SASjs */ - &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */ - &prefix._INIT_NUM /* initialisation time as numeric */ - &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format */ - &prefix.WORK /* avoid typing %sysfunc(pathname(work)) every time */ -; - -%let sasjs_prefix=&prefix; - -data _null_; - dttm=datetime(); - call symputx("&sasjs_prefix._init_num",dttm,'g'); - call symputx("&sasjs_prefix._init_dttm",put(dttm,E8601DT26.6),'g'); - call symputx("&sasjs_prefix.work",pathname('WORK'),'g'); -run; - -options - compress=CHAR /* default is none so ensure we have something! */ - datastmtchk=ALLKEYWORDS /* protection from overwriting input datasets */ - errorcheck=STRICT /* catch errs in libname/filename statements */ - fmterr /* ensure err when a format cannot be found */ - mergenoby=%str(ERR)OR /* throw err when a merge has no BY variables */ - missing=. /* changing this can cause hard to detect errs */ - noquotelenmax /* avoid warnings for long strings */ - noreplace /* avoid overwriting permanent datasets */ - ps=max /* reduce log size slightly */ - ls=max /* reduce log even more and avoid word truncation */ - validmemname=COMPATIBLE /* avoid special characters etc in table names */ - validvarname=V7 /* avoid special characters etc in variable names */ - varinitchk=%str(ERR)OR /* avoid data mistakes from variable name typos */ - varlenchk=%str(ERR)OR /* fail hard if truncation (data loss) can result */ -%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - noautocorrect /* disallow misspelled procedure names */ - dsoptions=note2err /* undocumented - convert bad NOTEs to ERRs */ -%end; -; - -%mend mp_init; diff --git a/009_macros/mp_jsonout.sas b/009_macros/mp_jsonout.sas deleted file mode 100644 index 2821601..0000000 --- a/009_macros/mp_jsonout.sas +++ /dev/null @@ -1,406 +0,0 @@ -/*** HELP START ***//** - @file mp_jsonout.sas - @brief Writes JSON in SASjs format to a fileref - @details This macro can be used to OPEN a JSON stream and send one or more - tables as arrays of rows, where each row can be an object or a nested array. - - There are two engines available - DATASTEP or PROCJSON. - - PROC JSON is fast but will produce errs like the ones below if - special chars are encountered. - - > (ERR)OR: Some code points did not transcode. - - > An object or array close is not valid at this point in the JSON text. - - > Date value out of range - - If this happens, try running with ENGINE=DATASTEP. - - The DATASTEP engine is used to handle special SAS missing numerics, and - can also convert entire datasets to formatted values. Output JSON is always - in UTF-8. - - Usage: - - filename tmp temp; - data class; set sashelp.class;run; - - %mp_jsonout(OPEN,jref=tmp) - %mp_jsonout(OBJ,class,jref=tmp) - %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y) - %mp_jsonout(CLOSE,jref=tmp) - - data _null_; - infile tmp; - input;putlog _infile_; - run; - - If you are building web apps with SAS then you are strongly encouraged to use - the mX_createwebservice macros in combination with the - [sasjs adapter](https://github.com/sasjs/adapter). - For more information see https://sasjs.io - - @param [in] action Valid values: - @li OPEN - opens the JSON - @li OBJ - sends a table with each row as an object - @li ARR - sends a table with each row in an array - @li CLOSE - closes the JSON - @param [in] ds The dataset to send. Must be a work table. - @param [out] jref= (_webout) The fileref to which to send the JSON - @param [out] dslabel= The name to give the table in the exported JSON - @param [in] fmt= (Y) Whether to keep (Y) or strip (N) formats from the table - @param [in] engine= (DATASTEP) Which engine to use to send the JSON. Options: - @li PROCJSON (default) - @li DATASTEP (more reliable when data has non standard characters) - @param [in] missing= (NULL) Special numeric missing values can be sent as NULL - (eg `null`) or as STRING values (eg `".a"` or `".b"`) - @param [in] showmeta= (N) Set to Y to output metadata alongside each table, - such as the column formats and types. The metadata is contained inside an - object with the same name as the table but prefixed with a dollar sign - ie, - `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}` - @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows - that should be converted to JSON - -

Related Files

- @li mp_ds2fmtds.sas - - @version 9.2 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ -%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y - ,engine=DATASTEP - ,missing=NULL - ,showmeta=N - ,maxobs=MAX -)/*/STORE SOURCE*/; -%local tempds colinfo fmtds i numcols numobs stmt_obs lastobs optval - tmpds1 tmpds2 tmpds3 tmpds4; -%let numcols=0; -%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); - -%if &action=OPEN %then %do; - options nobomfile; - data _null_;file &jref encoding='utf-8' lrecl=200; - put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"'; - run; -%end; -%else %if (&action=ARR or &action=OBJ) %then %do; - /* force variable names to always be uppercase in the JSON */ - options validvarname=upcase; - /* To avoid issues with _webout on EBI - such as encoding diffs and truncation - (https://support.sas.com/kb/49/325.html) we use temporary files */ - filename _sjs1 temp lrecl=200 ; - data _null_; file _sjs1 encoding='utf-8'; - put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; - run; - /* now write to _webout 1 char at a time */ - data _null_; - infile _sjs1 lrecl=1 recfm=n; - file &jref mod lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - filename _sjs1 clear; - - /* grab col defs */ - proc contents noprint data=&ds - out=_data_(keep=name type length format formatl formatd varnum label); - run; - %let colinfo=%scan(&syslast,2,.); - proc sort data=&colinfo; - by varnum; - run; - /* move meta to mac vars */ - data &colinfo; - if _n_=1 then call symputx('numcols',nobs,'l'); - set &colinfo end=last nobs=nobs; - name=upcase(name); - /* fix formats */ - if type=2 or type=6 then do; - typelong='char'; - length fmt $49.; - if format='' then fmt=cats('$',length,'.'); - else if formatl=0 then fmt=cats(format,'.'); - else fmt=cats(format,formatl,'.'); - end; - else do; - typelong='num'; - if format='' then fmt='best.'; - else if formatl=0 then fmt=cats(format,'.'); - else if formatd=0 then fmt=cats(format,formatl,'.'); - else fmt=cats(format,formatl,'.',formatd); - end; - /* 32 char unique name */ - newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27); - - call symputx(cats('name',_n_),name,'l'); - call symputx(cats('newname',_n_),newname,'l'); - call symputx(cats('length',_n_),length,'l'); - call symputx(cats('fmt',_n_),fmt,'l'); - call symputx(cats('type',_n_),type,'l'); - call symputx(cats('typelong',_n_),typelong,'l'); - call symputx(cats('label',_n_),coalescec(label,name),'l'); - /* overwritten when fmt=Y and a custom format exists in catalog */ - if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l'); - else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l'); - run; - - %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); - proc sql; - select count(*) into: lastobs from &ds; - %if &maxobs ne MAX %then %let lastobs=%sysfunc(min(&lastobs,&maxobs)); - - %if &engine=PROCJSON %then %do; - %if &missing=STRING %then %do; - %put &sysmacroname: Special Missings not supported in proc json.; - %put &sysmacroname: Switching to DATASTEP engine; - %goto datastep; - %end; - data &tempds; - set &ds; - &stmt_obs; - %if &fmt=N %then format _numeric_ best32.;; - /* PRETTY is necessary to avoid line truncation in large files */ - filename _sjs2 temp lrecl=131068 encoding='utf-8'; - proc json out=_sjs2 pretty - %if &action=ARR %then nokeys ; - ;export &tempds / nosastags fmtnumeric; - run; - /* send back to webout */ - data _null_; - infile _sjs2 lrecl=1 recfm=n; - file &jref mod lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - filename _sjs2 clear; - %end; - %else %if &engine=DATASTEP %then %do; - %datastep: - %if %sysfunc(exist(&ds)) ne 1 & %sysfunc(exist(&ds,VIEW)) ne 1 - %then %do; - %put &sysmacroname: &ds NOT FOUND!!!; - %return; - %end; - - %if &fmt=Y %then %do; - /** - * Extract format definitions - * First, by getting library locations from dictionary.formats - * Then, by exporting the width using proc format - * Cannot use maxw from sashelp.vformat as not always populated - * Cannot use fmtinfo() as not supported in all flavours - */ - %let tmpds1=%substr(fmtsum%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); - %let tmpds2=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); - %let tmpds3=%substr(cntl%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); - %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); - proc sql noprint; - create table &tmpds1 as - select cats(libname,'.',memname) as FMTCAT, - FMTNAME - from dictionary.formats - where fmttype='F' and libname is not null - and fmtname in (select format from &colinfo where format is not null) - order by 1; - create table &tmpds2( - FMTNAME char(32), - LENGTH num - ); - %local catlist cat fmtlist i; - select distinct fmtcat into: catlist separated by ' ' from &tmpds1; - %do i=1 %to %sysfunc(countw(&catlist,%str( ))); - %let cat=%scan(&catlist,&i,%str( )); - proc sql; - select distinct fmtname into: fmtlist separated by ' ' - from &tmpds1 where fmtcat="&cat"; - proc format lib=&cat cntlout=&tmpds3(keep=fmtname length); - select &fmtlist; - run; - proc sql; - insert into &tmpds2 select distinct fmtname,length from &tmpds3; - %end; - - proc sql; - create table &tmpds4 as - select a.*, b.length as MAXW - from &colinfo a - left join &tmpds2 b - on cats(a.format)=cats(upcase(b.fmtname)) - order by a.varnum; - data _null_; - set &tmpds4; - if not missing(maxw); - call symputx( - cats('fmtlen',_n_), - /* vars need extra padding due to JSON escaping of special chars */ - min(32767,ceil((max(length,maxw)+10)*1.5)) - ,'l' - ); - run; - - /* configure varlenchk - as we are explicitly shortening the variables */ - %let optval=%sysfunc(getoption(varlenchk)); - options varlenchk=NOWARN; - data _data_(compress=char); - /* shorten the new vars */ - length - %do i=1 %to &numcols; - &&name&i $&&fmtlen&i - %end; - ; - /* rename on entry */ - set &ds(rename=( - %do i=1 %to &numcols; - &&name&i=&&newname&i - %end; - )); - &stmt_obs; - - drop - %do i=1 %to &numcols; - &&newname&i - %end; - ; - %do i=1 %to &numcols; - %if &&typelong&i=num %then %do; - &&name&i=cats(put(&&newname&i,&&fmt&i)); - %end; - %else %do; - &&name&i=put(&&newname&i,&&fmt&i); - %end; - %end; - if _error_ then do; - call symputx('syscc',1012); - stop; - end; - run; - %let fmtds=&syslast; - options varlenchk=&optval; - %end; - - proc format; /* credit yabwon for special null removal */ - value bart (default=40) - %if &missing=NULL %then %do; - ._ - .z = null - %end; - %else %do; - ._ = [quote()] - . = null - .a - .z = [quote()] - %end; - other = [best.]; - - data &tempds; - attrib _all_ label=''; - %do i=1 %to &numcols; - %if &&typelong&i=char or &fmt=Y %then %do; - length &&name&i $&&fmtlen&i...; - format &&name&i $&&fmtlen&i...; - %end; - %end; - %if &fmt=Y %then %do; - set &fmtds; - %end; - %else %do; - set &ds; - %end; - &stmt_obs; - format _numeric_ bart.; - %do i=1 %to &numcols; - %if &&typelong&i=char or &fmt=Y %then %do; - if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do; - &&name&i='"'!!trim( - prxchange('s/"/\\"/',-1, /* double quote */ - prxchange('s/\x0A/\n/',-1, /* new line */ - prxchange('s/\x0D/\r/',-1, /* carriage return */ - prxchange('s/\x09/\\t/',-1, /* tab */ - prxchange('s/\x00/\\u0000/',-1, /* NUL */ - prxchange('s/\x0E/\\u000E/',-1, /* SS */ - prxchange('s/\x0F/\\u000F/',-1, /* SF */ - prxchange('s/\x01/\\u0001/',-1, /* SOH */ - prxchange('s/\x02/\\u0002/',-1, /* STX */ - prxchange('s/\x10/\\u0010/',-1, /* DLE */ - prxchange('s/\x11/\\u0011/',-1, /* DC1 */ - prxchange('s/\x1A/\\u001A/',-1, /* SUB */ - prxchange('s/\\/\\\\/',-1,&&name&i) - )))))))))))))!!'"'; - end; - else &&name&i=quote(cats(&&name&i)); - %end; - %end; - run; - - filename _sjs3 temp lrecl=131068 ; - data _null_; - file _sjs3 encoding='utf-8'; - if _n_=1 then put "["; - set &tempds; - if _n_>1 then put "," @; put - %if &action=ARR %then "[" ; %else "{" ; - %do i=1 %to &numcols; - %if &i>1 %then "," ; - %if &action=OBJ %then """&&name&i"":" ; - "&&name&i"n /* name literal for reserved variable names */ - %end; - %if &action=ARR %then "]" ; %else "}" ; ; - - /* close out the table */ - data _null_; - file _sjs3 mod encoding='utf-8'; - put ']'; - run; - data _null_; - infile _sjs3 lrecl=1 recfm=n; - file &jref mod lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - filename _sjs3 clear; - %end; - - proc sql; - drop table &colinfo, &tempds; - - %if %substr(&showmeta,1,1)=Y %then %do; - filename _sjs4 temp lrecl=131068 encoding='utf-8'; - data _null_; - file _sjs4; - length label $350; - put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; - do i=1 to &numcols; - name=quote(trim(symget(cats('name',i)))); - format=quote(trim(symget(cats('fmt',i)))); - label=quote(prxchange('s/\\/\\\\/',-1,trim(symget(cats('label',i))))); - length=quote(trim(symget(cats('length',i)))); - type=quote(trim(symget(cats('typelong',i)))); - if i>1 then put "," @@; - put name ':{"format":' format ',"label":' label - ',"length":' length ',"type":' type '}'; - end; - put '}}'; - run; - /* send back to webout */ - data _null_; - infile _sjs4 lrecl=1 recfm=n; - file &jref mod lrecl=1 recfm=n; - input sourcechar $char1. @@; - format sourcechar hex2.; - put sourcechar char1. @@; - run; - filename _sjs4 clear; - %end; -%end; - -%else %if &action=CLOSE %then %do; - data _null_; file &jref encoding='utf-8' mod ; - put "}"; - run; -%end; -%mend mp_jsonout; diff --git a/009_macros/mp_lib2cards.sas b/009_macros/mp_lib2cards.sas deleted file mode 100644 index 5319ea5..0000000 --- a/009_macros/mp_lib2cards.sas +++ /dev/null @@ -1,78 +0,0 @@ -/*** HELP START ***//** - @file - @brief Convert all library members to CARDS files - @details Gets list of members then calls the %mp_ds2cards() macro. - Usage: - - %mp_lib2cards(lib=sashelp - , outloc= C:\temp ) - - The output will be one cards file in the `outloc` directory per dataset in the - input `lib` library. If the `outloc` directory does not exist, it is created. - - To create a single SAS file with the first 1000 records of each table in a - library you could use this syntax: - - %mp_lib2cards(lib=sashelp - , outloc= /tmp - , outfile= myfile.sas - , maxobs= 1000 - ) - -

SAS Macros

- @li mf_mkdir.sas - @li mf_trimstr.sas - @li mp_ds2cards.sas - - @param [in] lib= () Library in which to convert all datasets - @param [out] outloc= (%sysfunc(pathname(work))) Location in which to store - output. No quotes. - @param [out] outfile= (0) Optional output file NAME - if provided, then - will create a single output file instead of one file per input table. - @param [in] maxobs= (max) limit output to the first maxobs rows - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_lib2cards(lib= - ,outloc=%sysfunc(pathname(work)) - ,maxobs=max - ,random_sample=NO - ,outfile=0 -)/*/STORE SOURCE*/; - -/* Find the tables */ -%local x ds memlist; -proc sql noprint; -select distinct lowcase(memname) - into: memlist - separated by ' ' - from dictionary.tables - where upcase(libname)="%upcase(&lib)"; - -/* trim trailing slash, if provided */ -%let outloc=%mf_trimstr(&outloc,/); -%let outloc=%mf_trimstr(&outloc,\); - -/* create the output directory */ -%mf_mkdir(&outloc) - -/* create the cards files */ -%do x=1 %to %sysfunc(countw(&memlist)); - %let ds=%scan(&memlist,&x); - %mp_ds2cards(base_ds=&lib..&ds - ,maxobs=&maxobs - ,random_sample=&random_sample - %if "&outfile" ne "0" %then %do; - ,append=YES - ,cards_file="&outloc/&outfile" - %end; - %else %do; - ,append=NO - ,cards_file="&outloc/&ds..sas" - %end; - ) -%end; - -%mend mp_lib2cards; diff --git a/009_macros/mp_lib2inserts.sas b/009_macros/mp_lib2inserts.sas deleted file mode 100644 index fcf88e9..0000000 --- a/009_macros/mp_lib2inserts.sas +++ /dev/null @@ -1,77 +0,0 @@ -/*** HELP START ***//** - @file - @brief Convert all data in a library to SQL insert statements - @details Gets list of members then calls the %mp_ds2inserts() - macro. - Usage: - - %mp_getddl(sashelp, schema=work, fref=tempref) - - %mp_lib2inserts(sashelp, schema=work, outref=tempref) - - %inc tempref; - - - The output will be one file in the outref fileref. - - -

SAS Macros

- @li mp_ds2inserts.sas - - - @param [in] lib Library in which to convert all datasets to inserts - @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid - options: - @li SAS (default) - suitable for regular proc sql - @li PGSQL - Used for Postgres databases - @param [in] maxobs= (max) The max number of observations (per table) to create - @param [out] outref= Output fileref in which to create the insert statements. - If it exists, it will be appended to, otherwise it will be created. - @param [out] schema= (0) The schema of the target database, or the libref. - @param [in] applydttm= (YES) If YES, any columns using datetime formats will - be converted to native DB datetime literals - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_lib2inserts(lib - ,flavour=SAS - ,outref=0 - ,schema=0 - ,maxobs=max - ,applydttm=YES -)/*/STORE SOURCE*/; - -/* Find the tables */ -%local x ds memlist; -proc sql noprint; -select distinct lowcase(memname) - into: memlist - separated by ' ' - from dictionary.tables - where upcase(libname)="%upcase(&lib)" - and memtype='DATA'; /* exclude views */ - - -%let flavour=%upcase(&flavour); -%if &flavour ne SAS and &flavour ne PGSQL %then %do; - %put %str(WAR)NING: &flavour is not supported; - %return; -%end; - - -/* create the inserts */ -%do x=1 %to %sysfunc(countw(&memlist)); - %let ds=%scan(&memlist,&x); - %mp_ds2inserts(&lib..&ds - ,outref=&outref - ,schema=&schema - ,outds=&ds - ,flavour=&flavour - ,maxobs=&maxobs - ,applydttm=&applydttm - ) -%end; - -%mend mp_lib2inserts; diff --git a/009_macros/mp_lockanytable.sas b/009_macros/mp_lockanytable.sas deleted file mode 100644 index 8c44d69..0000000 --- a/009_macros/mp_lockanytable.sas +++ /dev/null @@ -1,250 +0,0 @@ -/*** HELP START ***//** - @file - @brief Mechanism for locking tables to prevent parallel modifications - @details Uses a control table to enable ANY table to be locked for updates - (not just SAS datasets). - Only useful if every update uses the macro! Used heavily within - [Data Controller for SAS](https://datacontroller.io). - - @param [in] action The action to be performed. Valid values: - @li LOCK - Sets the lock flag, also confirms if a SAS lock is available - @li UNLOCK - Unlocks the table - @param [in] lib= (WORK) The libref of the table to lock. Should already be - assigned. - @param [in] ds= The dataset to lock - @param [in] ref= A meaningful reference to enable the lock to be traced. Max - length is 200 characters. - @param [out] ctl_ds= (0) The control table which controls the actual locking. - Should already be assigned and available. The definition is available by - running the mddl_dc_locktable.sas macro. - - @param [in] loops= (25) Number of times to check for a lock. - @param [in] loop_secs= (1) Seconds to wait between each lock attempt - -

SAS Macros

- @li mf_fmtdttm.sas - @li mp_abort.sas - @li mp_lockfilecheck.sas - @li mf_getuser.sas - -

Related Macros

- @li mp_lockanytable.test.sas - - @version 9.2 - -**//*** HELP END ***/ - -%macro mp_lockanytable( - action - ,lib= WORK - ,ds=0 - ,ref= - ,ctl_ds=0 - ,loops=25 - ,loop_secs=1 - ); -data _null_; - if _n_=1 then putlog "&sysmacroname entry vars:"; - set sashelp.vmacro; - where scope="&sysmacroname"; - put name '=' value; -run; - -%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE) - ,mac=&sysmacroname - ,msg=%str(dataset was not provided) -) -%mp_abort(iftrue= (&ctl_ds=0) - ,mac=&sysmacroname - ,msg=%str(Control dataset was not provided) -) - -/* set up lib & mac vars */ -%let lib=%upcase(&lib); -%let ds=%upcase(&ds); -%let action=%upcase(&action); -%local user x trans msg abortme; -%let user=%mf_getuser(); -%let abortme=0; - -%mp_abort(iftrue= (&action ne LOCK & &action ne UNLOCK & &action ne MAKETABLE) - ,mac=&sysmacroname - ,msg=%str(Invalid action (&action) provided) -) - -/* if an err condition exists, exit before we even begin */ -%mp_abort(iftrue= (&syscc>0 and &action=LOCK) - ,mac=&sysmacroname - ,msg=%str(aborting due to syscc=&syscc on LOCK entry) -) - -/* do not bother locking work tables (else may affect all WORK libraries) */ -%if (%upcase(&lib)=WORK or %str(&lib)=%str()) & &action ne MAKETABLE %then %do; - %put NOTE: WORK libraries will not be registered in the locking system.; - %return; -%end; - -/* do not proceed if no observations can be processed */ -%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0) - ,mac=&sysmacroname - ,msg=%str(cannot continue when options obs = 0) -) - -%if &ACTION=LOCK %then %do; - - /* abort if a SAS lock is already in place, or cannot be applied */ - %mp_lockfilecheck(&lib..&ds) - - /* next, check there is a record for this table */ - %local record_exists_check; - proc sql noprint; - select count(*) into: record_exists_check from &ctl_ds - where LOCK_LIB ="&lib" and LOCK_DS="&ds"; - quit; - %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc; - %if &record_exists_check=0 %then %do; - data _null_; - putlog "&sysmacroname: adding record to lock table.."; - run; - - data ; - if 0 then set &ctl_ds; - LOCK_LIB ="&lib"; - LOCK_DS="&ds"; - LOCK_STATUS_CD='LOCKED'; - LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt; - LOCK_USER_NM="&user"; - LOCK_PID="&sysjobid"; - LOCK_REF="&ref"; - output;stop; - run; - %let trans=&syslast; - proc append base=&ctl_ds data=&trans; - run; - %end; - /* if record does exist, perform lock attempts */ - %else %do x=1 %to &loops; - data _null_; - putlog "&sysmacroname: attempting lock (iteration &x) "@; - putlog "at %sysfunc(datetime(),datetime19.) .."; - run; - - proc sql; - update &ctl_ds - set LOCK_STATUS_CD='LOCKED' - , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt - , LOCK_USER_NM="&user" - , LOCK_PID="&sysjobid" - , LOCK_REF="&ref" - where LOCK_LIB ="&lib" and LOCK_DS="&ds"; - quit; - /** - * NOTE - occasionally SQL server will return an err code (deadlocked - * transaction). If so, ignore it, keep calm, and carry on.. - */ - %if &syscc>0 %then %do; - data _null_; - putlog 'NOTE-' / 'NOTE-'; - putlog "NOTE- &sysmacroname: Update failed. "@; - putlog "Resetting err conditions and re-attempting."; - putlog "NOTE- syscc=&syscc syserr=&syserr sqlrc=&sqlrc"; - putlog 'NOTE-' / 'NOTE-'; - run; - %let syscc=0; - %let sqlrc=0; - %end; - - /* now check if the record was successfully updated */ - %local success_check; - proc sql noprint; - select count(*) into: success_check from &ctl_ds - where LOCK_LIB ="&lib" and LOCK_DS="&ds" - and LOCK_PID="&sysjobid" and LOCK_STATUS_CD='LOCKED'; - quit; - %if &success_check=0 %then %do; - %if &x < &loops %then %do; - /* pause before next check */ - data _null_; - putlog 'NOTE-' / 'NOTE-'; - putlog "NOTE- &sysmacroname: table locked, waiting "@; - putlog "%sysfunc(sleep(&loop_secs)) seconds.. "; - putlog "NOTE- (iteration &x of &loops)"; - putlog 'NOTE-' / 'NOTE-'; - run; - %end; - %else %do; - %let msg=Unable to lock &lib..&ds via &ctl_ds after &loops attempts.\n - Please ask your administrator to investigate!; - %let abortme=1; - %end; - %end; - %else %do; - data _null_; - putlog 'NOTE-' / 'NOTE-'; - putlog "NOTE- &sysmacroname: Table &lib..&ds locked at "@; - putlog " %sysfunc(datetime(),datetime19.) (iteration &x)"@; - putlog 'NOTE-' / 'NOTE-'; - run; - %if &syscc>0 %then %do; - %put setting syscc(&syscc) back to 0; - %let syscc=0; - %end; - %let x=&loops; /* no more iterations needed */ - %end; - %end; -%end; -%else %if &ACTION=UNLOCK %then %do; - %local status cnt; - %let cnt=0; - proc sql noprint; - select count(*) into: cnt from &ctl_ds where LOCK_LIB ="&lib" & LOCK_DS="&ds"; - %if &cnt=0 %then %do; - %put %str(WAR)NING: &lib..&ds was not previously locked in &ctl_ds!; - %end; - %else %do; - select LOCK_STATUS_CD into: status from &ctl_ds - where LOCK_LIB ="&lib" and LOCK_DS="&ds"; - quit; - %if &syscc>0 %then %put syscc=&syscc sqlrc=&sqlrc; - %if &status=LOCKED %then %do; - data _null_; - putlog "&sysmacroname: unlocking &lib..&ds:"; - run; - proc sql; - update &ctl_ds - set LOCK_STATUS_CD='UNLOCKED' - , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt - , LOCK_USER_NM="&user" - , LOCK_PID="&sysjobid" - , LOCK_REF="&ref" - where LOCK_LIB ="&lib" and LOCK_DS="&ds"; - quit; - %end; - %else %if &status=UNLOCKED %then %do; - %put %str(WAR)NING: &lib..&ds is already unlocked!; - %end; - %else %do; - %put NOTE: Unrecognised STATUS_CD (&status) in &ctl_ds; - %let abortme=1; - %end; - %end; -%end; -%else %do; - %let msg=lock_anytable given unsupported action (&action); - %let abortme=1; -%end; - -/* catch errs - mp_abort must be called outside of a logic block */ -%mp_abort(iftrue=(&abortme=1), - msg=%superq(msg), - mac=&sysmacroname -) - -%exit_macro: -data _null_; - put "&sysmacroname: Exit vars: action=&action lib=&lib ds=&ds"; - put " syscc=&syscc sqlrc=&sqlrc syserr=&syserr"; -run; -%mend mp_lockanytable; - - diff --git a/009_macros/mp_lockfilecheck.sas b/009_macros/mp_lockfilecheck.sas deleted file mode 100644 index 3125fa0..0000000 --- a/009_macros/mp_lockfilecheck.sas +++ /dev/null @@ -1,103 +0,0 @@ -/*** HELP START ***//** - @file - @brief Aborts if a SAS lock file is in place, or if one cannot be applied. - @details Used in conjuction with the mp_lockanytable macro. - More info here: https://sasensei.com/flash/24 - - Usage: - - data work.test; a=1;run; - %mp_lockfilecheck(work.test) - - @param [in] libds The libref.dataset for which to check the lock status - -

SAS Macros

- @li mp_abort.sas - @li mf_getattrc.sas - -

Related Macros

- @li mp_lockanytable.sas - @li mp_lockfilecheck.test.sas - - @version 9.2 -**//*** HELP END ***/ - -%macro mp_lockfilecheck( - libds -)/*/STORE SOURCE*/; - -data _null_; - if _n_=1 then putlog "&sysmacroname entry vars:"; - set sashelp.vmacro; - where scope="&sysmacroname"; - put name '=' value; -run; - -%mp_abort(iftrue= (&syscc>0) - ,mac=checklock.sas - ,msg=Aborting with syscc=&syscc on entry. -) -%mp_abort(iftrue= ("&libds"="0") - ,mac=&sysmacroname - ,msg=%str(libds not provided) -) - -%local msg lib ds; -%let lib=%upcase(%scan(&libds,1,.)); -%let ds=%upcase(%scan(&libds,2,.)); - -/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */ -%if %scan(&libds,2,-)=FC %then %do; - %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds; - %return; -%end; - -/* do not proceed if no observations can be processed */ -%let msg=options obs = 0. syserrortext=%superq(syserrortext); -%mp_abort(iftrue= (%sysfunc(getoption(OBS))=0) - ,mac=checklock.sas - ,msg=%superq(msg) -) - -data _null_; - putlog "Checking engine & member type"; -run; -%local engine memtype; -%let memtype=%mf_getattrc(&libds,MTYPE); -%let engine=%mf_getattrc(&libds,ENGINE); - -%if &engine ne V9 and &engine ne BASE %then %do; - data _null_; - putlog "Lib &lib is not assigned using BASE engine - uses &engine instead"; - putlog "SAS lock check will not be performed"; - run; - %return; -%end; -%else %if &memtype ne DATA %then %do; - %put NOTE: Cannot lock a VIEW!! Memtype=&memtype; - %return; -%end; - -data _null_; - putlog "Engine = &engine, memtype=&memtype"; - putlog "Attempting lock statement"; -run; - -lock &libds; - -%local abortme; -%let abortme=0; -%if &syscc>0 or &SYSLCKRC ne 0 %then %do; - %let msg=Unable to apply lock on &libds (SYSLCKRC=&SYSLCKRC syscc=&syscc); - %put %str(ERR)OR: &sysmacroname: &msg; - %let abortme=1; -%end; - -lock &libds clear; - -%mp_abort(iftrue= (&abortme=1) - ,mac=&sysmacroname - ,msg=%superq(msg) -) - -%mend mp_lockfilecheck; diff --git a/009_macros/mp_makedata.sas b/009_macros/mp_makedata.sas deleted file mode 100644 index d08b192..0000000 --- a/009_macros/mp_makedata.sas +++ /dev/null @@ -1,107 +0,0 @@ -/*** HELP START ***//** - @file - @brief Create sample data based on the structure of an empty table - @details Many SAS projects involve sensitive datasets. One way to _ensure_ - the data is anonymised, is never to receive it in the first place! Often - consultants are provided with empty tables, and expected to create complex - ETL flows. - - This macro can help by taking an empty table, and populating it with data - according to the variable types and formats. - - TODO: - @li Consider dates, datetimes, times, integers etc - - Usage: - - proc sql; - create table work.example( - TX_FROM float format=datetime19., - DD_TYPE char(16), - DD_SOURCE char(2048), - DD_SHORTDESC char(256), - constraint pk primary key(tx_from, dd_type,dd_source), - constraint nnn not null(DD_SHORTDESC) - ); - %mp_makedata(work.example) - - @param [in] libds The empty table (libref.dataset) in which to create data - @param [out] obs= (500) The maximum number of records to create. The table - is sorted with nodup on the primary key, so the actual number of records may - be lower than this. - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_getvarlen.sas - @li mf_getvarlist.sas - @li mf_islibds.sas - @li mf_nobs.sas - @li mp_getcols.sas - @li mp_getpk.sas - -

Related Macros

- @li mp_makedata.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_makedata(libds - ,obs=500 - ,seed=1 -)/*/STORE SOURCE*/; - -%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk; - -%if %mf_islibds(&libds)=0 %then %do; - %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format; - %return; -%end; -%else %if %mf_nobs(&libds)>0 %then %do; - %put &sysmacroname: &libds has data, it will not be recreated; - %return; -%end; - -/* set up temporary vars */ -%let ds1=%mf_getuniquename(prefix=mp_makedatads1); -%let ds2=%mf_getuniquename(prefix=mp_makedatads2); -%let lib=%scan(&libds,1,.); -%let ds=%scan(&libds,2,.); - -/* grab the primary key vars */ -%mp_getpk(&lib,ds=&ds,outds=&ds1) - -proc sql noprint; -select coalescec(pk_fields,'_all_') into: pk_fields from &ds1; - -data &ds2; - if 0 then set &libds; - do _n_=1 to &obs; - %let charvars=%mf_getvarlist(&libds,typefilter=C); - %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars)); - %let col=%scan(&charvars,&i); - /* create random value based on observation number and colum length */ - &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32); - %end; - - %let numvars=%mf_getvarlist(&libds,typefilter=N); - %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars)); - %let col=%scan(&numvars,&i); - &col=_n_; - %end; - output; - end; - stop; -run; -proc sort data=&ds2 nodupkey; - by &pk_fields; -run; - -proc append base=&libds data=&ds2; -run; - -proc sql; -drop table &ds1, &ds2; - -%mend mp_makedata; diff --git a/009_macros/mp_md5.sas b/009_macros/mp_md5.sas deleted file mode 100644 index 85ca781..0000000 --- a/009_macros/mp_md5.sas +++ /dev/null @@ -1,57 +0,0 @@ -/*** HELP START ***//** - @file - @brief Generates an md5 expression for hashing a set of variables - @details This is the same algorithm used to hash records in - [Data Controller for SAS](https://datacontroller.io). - - It is not designed to be efficient - it is designed to be effective, - given the range of edge cases (large floating points, special missing - numerics, thousands of columns, very wide columns). - - It can be used only in data step, eg as follows: - - data _null_; - set sashelp.class; - hashvar=%mp_md5(cvars=name sex, nvars=age height weight); - put hashvar=; - run; - - Unfortunately it will not run in SQL - it fails with the following message: - - > The width value for HEX is out of bounds. It should be between 1 and 16 - - The macro will also cause errors if the data contains (non-special) missings - and the (undocumented) `options dsoptions=nonote2err;` is in effect. - - This can be avoided in two ways: - - @li Global option: `options dsoptions=nonote2err;` - @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;` - - @param [in] cvars= () Space seperated list of character variables - @param [in] nvars= () Space seperated list of numeric variables - -

Related Programs

- @li mp_init.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_md5(cvars=,nvars=); -%local i var sep; -put(md5( - %do i=1 %to %sysfunc(countw(&cvars)); - %let var=%scan(&cvars,&i,%str( )); - &sep put(md5(trim(&var)),$hex32.) - %let sep=!!; - %end; - %do i=1 %to %sysfunc(countw(&nvars)); - %let var=%scan(&nvars,&i,%str( )); - /* multiply by 1 to strip precision errors (eg 0 != 0) */ - /* but ONLY if not missing, else will lose any special missing values */ - &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.) - %let sep=!!; - %end; -),$hex32.) -%mend mp_md5; diff --git a/009_macros/mp_perflog.sas b/009_macros/mp_perflog.sas deleted file mode 100644 index 0ccd0c8..0000000 --- a/009_macros/mp_perflog.sas +++ /dev/null @@ -1,43 +0,0 @@ -/*** HELP START ***//** - @file - @brief Logs a message in a dataset every time it is invoked - @details If the dataset does not exist, it is created. - Usage: - - %mp_perflog(started) - %mp_perflog() - %mp_perflog(startanew,libds=work.newdataset) - %mp_perflog(finished,libds=work.newdataset) - %mp_perflog(finished) - - - @param [in] label Provide label to go into the control dataset - @param [in] libds= (work.mp_perflog) Provide a dataset in which to store - performance stats. Default name is work.mp_perflog; - - @version 9.2 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_perflog(label,libds=work.mp_perflog -)/*/STORE SOURCE*/; - - %if not (%mf_existds(&libds)) %then %do; - data &libds; - length sysjobid $10 label $256 dttm 8.; - format dttm datetime19.3; - call missing(of _all_); - stop; - run; - %end; - - proc sql; - insert into &libds - set sysjobid="&sysjobid" - ,label=symget('label') - ,dttm=%sysfunc(datetime()); - quit; - -%mend mp_perflog; diff --git a/009_macros/mp_prevobs.sas b/009_macros/mp_prevobs.sas deleted file mode 100644 index b4b06a3..0000000 --- a/009_macros/mp_prevobs.sas +++ /dev/null @@ -1,88 +0,0 @@ -/*** HELP START ***//** - @file - @brief Enables previous observations to be re-instated - @details Remembers the last X observations by storing them in a hash table. - Is a convenience over the use of lag() or retain, when an entire observation - needs to be restored. - - This macro will also restore automatic variables (such as _n_ and _error_). - - Example Usage: - - data example; - set sashelp.class; - calc_var=_n_*3; - %* initialise hash and save from PDV ; - %mp_prevobs(INIT,history=2) - if _n_ =10 then do; - %* fetch previous but 1 record; - %mp_prevobs(FETCH,-2) - put _n_= name= age= calc_var=; - %* fetch previous record; - %mp_prevobs(FETCH,-1) - put _n_= name= age= calc_var=; - %* reinstate current record ; - %mp_prevobs(FETCH,0) - put _n_= name= age= calc_var=; - end; - run; - - Result: - - mp_prevobs sas - - Credit is made to `data _null_` for authoring this very helpful paper: - https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf - - @param [in] action Either FETCH a current or previous record, or INITialise. - @param [in] record The relative (to current) position of the previous row - to return. - @param [in] history= (5) The number of records to retain in the hash table. - @param [in] prefix= (mp_prevobs) The prefix to give to the variables used to - store the hash name and index. - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_prevobs(action,record,history=5,prefix=mp_prevobs -)/*/STORE SOURCE*/; -%let action=%upcase(&action); -%let prefix=%upcase(&prefix); -%let record=%eval((&record+0) * -1); - -%if &action=INIT %then %do; - - if _n_ eq 1 then do; - attrib &prefix._VAR length=$64; - dcl hash &prefix._HASH(ordered:'Y'); - &prefix._KEY=0; - &prefix._HASH.defineKey("&prefix._KEY"); - do while(1); - call vnext(&prefix._VAR); - if &prefix._VAR='' then leave; - if &prefix._VAR eq "&prefix._VAR" then continue; - else if &prefix._VAR eq "&prefix._KEY" then continue; - &prefix._HASH.defineData(&prefix._VAR); - end; - &prefix._HASH.defineDone(); - end; - /* this part has to happen before FETCHing */ - &prefix._KEY+1; - &prefix._rc=&prefix._HASH.add(); - if &prefix._rc then putlog 'adding' &prefix._rc=; - %if &history>0 %then %do; - if &prefix._key>&history+1 then - &prefix._HASH.remove(key: &prefix._KEY - &history - 1); - if &prefix._rc then putlog 'removing' &prefix._rc=; - %end; -%end; -%else %if &action=FETCH %then %do; - if &record>&prefix._key then putlog "Not enough records in &Prefix._hash yet"; - else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record); - if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY= - "with record &record and " _n_=; -%end; - -%mend mp_prevobs; diff --git a/009_macros/mp_recursivejoin.sas b/009_macros/mp_recursivejoin.sas deleted file mode 100644 index 4a15aa9..0000000 --- a/009_macros/mp_recursivejoin.sas +++ /dev/null @@ -1,91 +0,0 @@ -/*** HELP START ***//** - @file - @brief Returns all children from a hierarchy table for a specified parent - @details Where data stores hierarchies in a simple parent / child mapping, - it is not always straightforward to extract all the children for a - particular parent. This problem is known as a recursive self join. This - macro will extract all the descendents for a parent. - Usage: - - data have; - p=1;c=2;output; - p=2;c=3;output; - p=2;c=4;output; - p=3;c=5;output; - p=6;c=7;output; - p=8;c=9;output; - run; - - %mp_recursivejoin(base_ds=have - ,outds=want - ,matchval=1 - ,parentvar=p - ,childvar=c - ) - - @param [in] base_ds= base table containing hierarchy (not modified) - @param [out] outds= the output dataset to create with the generated hierarchy - @param [in] matchval= the ultimate parent from which to filter - @param [in] parentvar= name of the parent variable - @param [in] childvar= () name of the child variable (should be same type as - parent) - @param [in] mdebug= set to 1 to prevent temp tables being dropped - - - @returns outds contains the following variables: - - level (0 = top level) - - &parentvar - - &childvar (null if none found) - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_recursivejoin(base_ds= - ,outds= - ,matchval= - ,parentvar= - ,childvar= - ,iter= /* reserved for internal / recursive use by the macro itself */ - ,maxiter=500 /* avoid infinite loop */ - ,mDebug=0); - -%if &iter= %then %do; - proc sql; - create table &outds as - select 0 as level,&parentvar, &childvar - from &base_ds - where &parentvar=&matchval; - %if &sqlobs.=0 %then %do; - %put NOTE: &sysmacroname: No match for &parentvar=&matchval; - %return; - %end; - %let iter=1; -%end; -%else %if &iter>&maxiter %then %return; - -proc sql; -create table _data_ as - select &iter as level - ,curr.&childvar as &parentvar - ,base_ds.&childvar as &childvar - from &outds curr - left join &base_ds base_ds - on curr.&childvar=base_ds.&parentvar - where curr.level=%eval(&iter.-1) - & curr.&childvar is not null; -%local append_ds; %let append_ds=&syslast; -%local obs; %let obs=&sqlobs; -insert into &outds select distinct * from &append_ds; -%if &mdebug=0 %then drop table &append_ds;; - -%if &obs %then %do; - %mp_recursivejoin(iter=%eval(&iter.+1) - ,outds=&outds,parentvar=&parentvar - ,childvar=&childvar - ,base_ds=&base_ds - ) -%end; - -%mend mp_recursivejoin; diff --git a/009_macros/mp_replace.sas b/009_macros/mp_replace.sas deleted file mode 100644 index fd391f4..0000000 --- a/009_macros/mp_replace.sas +++ /dev/null @@ -1,152 +0,0 @@ -/*** HELP START ***//** - @file - @brief Performs a text substitution on a file - @details Performs a find and replace on a file, either in place or to a new - file. Can be used on files where lines are longer than 32767. - - Works by reading in the file byte by byte, then marking the beginning and end - of each matched string, before finally doing the replace. - - Full credit for this highly efficient and syntactically satisfying SAS logic - goes to [Bartosz Jabłoński](https://www.linkedin.com/in/yabwon), founder of - the [SAS Packages](https://github.com/yabwon/SAS_PACKAGES) framework. - - Usage: - - %let file="%sysfunc(pathname(work))/file.txt"; - %let str=replace/me; - %let rep=with/this; - data _null_; - file &file; - put 'blahblah'; - put "blahblah&str.blah"; - put 'blahblahblah'; - run; - %mp_replace(&file, findvar=str, replacevar=rep) - data _null_; - infile &file; - input; - list; - run; - - Note - if you are running a version of SAS that will allow the io package in - LUA, you can also use this macro: mp_gsubfile.sas - - @param [in] infile The QUOTED path to the file on which to perform the - substitution - @param [in] findvar= Macro variable NAME containing the string to search for - @param [in] replacevar= Macro variable NAME containing the replacement string - @param [out] outfile= (0) Optional QUOTED path to the adjusted output file (to - avoid overwriting the first file). - -

SAS Macros

- @li mf_getuniquefileref.sas - @li mf_getuniquename.sas - -

Related Macros

- @li mp_chop.sas - @li mp_gsubfile.sas - @li mp_replace.test.sas - - @version 9.4 - @author Bartosz Jabłoński - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_replace(infile, - findvar=, - replacevar=, - outfile=0 -)/*/STORE SOURCE*/; - -%local inref dttm ds1; -%let inref=%mf_getuniquefileref(); -%let outref=%mf_getuniquefileref(); -%if &outfile=0 %then %let outfile=&infile; -%let ds1=%mf_getuniquename(prefix=allchars); -%let ds2=%mf_getuniquename(prefix=startmark); - -/* START */ -%let dttm=%sysfunc(datetime()); - -filename &inref &infile lrecl=1 recfm=n; - -data &ds1; - infile &inref; - input sourcechar $char1. @@; - format sourcechar hex2.; -run; - -data &ds2; - /* set find string to length in bytes to cover trailing spaces */ - length string $ %length(%superq(&findvar)); - string =symget("&findvar"); - drop string; - - firstchar=char(string,1); - findlen=lengthm(string); /* <- for trailing bytes */ - - do _N_=1 to nobs; - set &ds1 nobs=nobs point=_N_; - if sourcechar=firstchar then do; - pos=1; - s=0; - do point=_N_ to min(_N_ + findlen -1,nobs); - set &ds1 point=point; - if sourcechar=char(string, pos) then s + 1; - else goto _leave_; - pos+1; - end; - _leave_: - if s=findlen then do; - START =_N_; - _N_ =_N_+ s - 1; - STOP =_N_; - output; - end; - end; - end; - stop; - keep START STOP; -run; - -data &ds1; - declare hash HS(dataset:"&ds2(keep=start)"); - HS.defineKey("start"); - HS.defineDone(); - declare hash HE(dataset:"&ds2(keep=stop)"); - HE.defineKey("stop"); - HE.defineDone(); - do until(eof); - set &ds1 end=eof curobs =n; - start = ^HS.check(key:n); - stop = ^HE.check(key:n); - length strt $ 1; - strt =put(start,best. -L); - retain out 1; - if out then output; - if start then out=0; - if stop then out=1; - end; - stop; - keep sourcechar strt; -run; - -filename &outref &outfile recfm=n; - -data _null_; - length replace $ %length(%superq(&replacevar)); - replace=symget("&replacevar"); - file &outref; - do until(eof); - set &ds1 end=eof; - if strt ="1" then put replace char.; - else put sourcechar char1.; - end; - stop; -run; - -/* END */ -%put &sysmacroname took %sysevalf(%sysfunc(datetime())-&dttm) seconds to run; - -%mend mp_replace; diff --git a/009_macros/mp_reseterror.sas b/009_macros/mp_reseterror.sas deleted file mode 100644 index c89cd79..0000000 --- a/009_macros/mp_reseterror.sas +++ /dev/null @@ -1,27 +0,0 @@ -/*** HELP START ***//** - @file - @brief Reset when an err condition occurs - @details When building apps, sometimes an operation must be attempted that - can cause an err condition. There is no try catch in SAS! So the err state - must be caught and reset. - - This macro attempts to do that reset. - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_reseterror( -)/*/STORE SOURCE*/; - -options obs=max replace nosyntaxcheck; -%let syscc=0; - -%if "&sysprocessmode " = "SAS Stored Process Server " %then %do; - data _null_; - rc=stpsrvset('program error', 0); - run; -%end; - -%mend mp_reseterror; diff --git a/009_macros/mp_resetoption.sas b/009_macros/mp_resetoption.sas deleted file mode 100644 index 0615214..0000000 --- a/009_macros/mp_resetoption.sas +++ /dev/null @@ -1,39 +0,0 @@ -/*** HELP START ***//** - @file - @brief Reset an option to original value - @details Inspired by the SAS Jedi - -https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options - - Called as follows: - - options obs=30 ps=max; - %mp_resetoption(OBS) - %mp_resetoption(PS) - - - @param [in] option the option to reset - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_resetoption(option /* the option to reset */ -)/*/STORE SOURCE*/; - -%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do; - data _null_; - length code $1500; - startup=getoption("&option",'startupvalue'); - current=getoption("&option"); - if startup ne current then do; - code =cat('OPTIONS ',getoption("&option",'keyword','startupvalue'),';'); - putlog "NOTE: Resetting system option: " code ; - call execute(code ); - end; - run; -%end; -%else %do; - %put &sysmacroname: reset option feature unavailable on &sysvlong; -%end; -%mend mp_resetoption; diff --git a/009_macros/mp_retainedkey.sas b/009_macros/mp_retainedkey.sas deleted file mode 100644 index 9fbad99..0000000 --- a/009_macros/mp_retainedkey.sas +++ /dev/null @@ -1,247 +0,0 @@ -/*** HELP START ***//** - @file - @brief Generate and apply retained key values to a staging table - @details This macro will populate a staging table with a Retained Key based on - a business key and a base (target) table. - - Definition of retained key ([source]( - http://bukhantsov.org/2012/04/what-is-data-vault/)): - - > The retained key is a key which is mapped to business key one-to-one. In - > comparison, the surrogate key includes time and there can be many surrogate - > keys corresponding to one business key. This explains the name of the key, - > it is retained with insertion of a new version of a row while surrogate key - > is increasing. - - This macro is designed to be used as part of a wider load / ETL process (such - as the one in [Data Controller for SAS](https://datacontroller.io)). - - Specifically, the macro assumes that the base table has already been 'locked' - (eg with the mp_lockanytable.sas macro) prior to invocation. Also, several - tables are assumed to exist (names are configurable): - - @li work.staging_table - the staged data, minus the retained key element - @li permlib.base_table - the target table to be loaded (**not** loaded by this - macro) - @li permlib.maxkeytable - optional, used to store load metaadata. - The definition is available by running mp_coretable.sas as follows: - `mp_coretable(MAXKEYTABLE)`. - @li permlib.locktable - Necessary if maxkeytable is being populated. The - definition is available by running mp_coretable.sas as follows: - `mp_coretable(LOCKTABLE)`. - - - @param [in] base_lib= (WORK) Libref of the base (target) table. - @param [in] base_dsn= (BASETABLE) Name of the base (target) table. - @param [in] append_lib= (WORK) Libref of the staging table - @param [in] append_dsn= (APPENDTABLE) Name of the staging table - @param [in] retained_key= (DEFAULT_RK) Name of RK to generate (should exist on - base table) - @param [in] business_key= (PK1 PK2) Business key against which to generate - RK values. Should be unique and not null on the staging table. - @param [in] check_uniqueness=(NO) Set to yes to perform a uniqueness check. - Recommended if there is a chance that the staging data is not unique on the - business key. - @param [in] maxkeytable= (0) Provide a maxkeytable libds reference here, to - store load metadata (maxkey val, load time). Set to zero if metadata is not - required, eg, when preparing a 'dummy' load. Structure is described above. - See below for sample data. - |KEYTABLE:$32.|KEYCOLUMN:$32.|MAX_KEY:best.|PROCESSED_DTTM:E8601DT26.6| - |---|---|---|---| - |`DC487173.MPE_SELECTBOX `|`SELECTBOX_RK `|`55 `|`1950427787.8 `| - |`DC487173.MPE_FILTERANYTABLE `|`filter_rk `|`14 `|`1951053886.8 `| - @param [in] locktable= (0) If updating the maxkeytable, provide the libds - reference to the lock table (per mp_lockanytable.sas macro) - @param [in] filter_str= Apply a filter - useful for SCD2 or BITEMPORAL loads. - Example: `filter_str=%str( (where=( &now < &tech_to)) )` - @param [out] outds= (WORK.APPEND) Output table (staging table + retained key) - -

SAS Macros

- @li mf_existvar.sas - @li mf_fmtdttm.sas - @li mf_getquotedstr.sas - @li mf_getuniquename.sas - @li mf_nobs.sas - @li mp_abort.sas - @li mp_lockanytable.sas - -

Related Macros

- @li mp_filterstore.sas - @li mp_retainedkey.test.sas - - @version 9.2 - -**//*** HELP END ***/ - -%macro mp_retainedkey( - base_lib=WORK - ,base_dsn=BASETABLE - ,append_lib=WORK - ,append_dsn=APPENDTABLE - ,retained_key=DEFAULT_RK - ,business_key= PK1 PK2 - ,check_uniqueness=NO - ,maxkeytable=0 - ,locktable=0 - ,outds=WORK.APPEND - ,filter_str= -); -%put &sysmacroname entry vars:; -%put _local_; - -%local base_libds app_libds key_field check maxkey idx_pk newkey_cnt iserr - msg x tempds1 tempds2 comma_pk appnobs checknobs dropvar tempvar idx_val; -%let base_libds=%upcase(&base_lib..&base_dsn); -%let app_libds=%upcase(&append_lib..&append_dsn); -%let tempds1=%mf_getuniquename(); -%let tempds2=%mf_getuniquename(); -%let comma_pk=%mf_getquotedstr(in_str=%str(&business_key),dlm=%str(,),quote=); -%let outds=%sysfunc(ifc(%index(&outds,.)=0,work.&outds,&outds)); -/* validation checks */ -%let iserr=0; -%if &syscc>0 %then %do; - %let iserr=1; - %let msg=%str(SYSCC=&syscc on macro entry); -%end; -%else %if %sysfunc(exist(&base_libds))=0 %then %do; - %let iserr=1; - %let msg=%str(Base LIBDS (&base_libds) expected but NOT FOUND); -%end; -%else %if %sysfunc(exist(&app_libds))=0 %then %do; - %let iserr=1; - %let msg=%str(Append LIBDS (&app_libds) expected but NOT FOUND); -%end; -%else %if &maxkeytable ne 0 and %sysfunc(exist(&maxkeytable))=0 %then %do; - %let iserr=1; - %let msg=%str(Maxkeytable (&maxkeytable) expected but NOT FOUND); -%end; -%else %if &maxkeytable ne 0 and %sysfunc(exist(&locktable))=0 %then %do; - %let iserr=1; - %let msg=%str(Locktable (&locktable) expected but NOT FOUND); -%end; -%else %if %length(&business_key)=0 %then %do; - %let iserr=1; - %let msg=%str(Business key (&business_key) expected but NOT FOUND); -%end; - -%do x=1 %to %sysfunc(countw(&business_key)); - /* check business key values exist */ - %let key_field=%scan(&business_key,&x,%str( )); - %if not %mf_existvar(&app_libds,&key_field) %then %do; - %let iserr=1; - %let msg=Business key (&key_field) not found on &app_libds!; - %goto err; - %end; - %else %if not %mf_existvar(&base_libds,&key_field) %then %do; - %let iserr=1; - %let msg=Business key (&key_field) not found on &base_libds!; - %goto err; - %end; -%end; -%err: -%if &iserr=1 %then %do; - /* err case so first perform an unlock of the base table before exiting */ - %mp_lockanytable( - UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable - ) -%end; -%mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg)) - -proc sql noprint; -select sum(max(&retained_key),0) into: maxkey from &base_libds; - -/** - * get base table RK and bus field values for lookup - */ -proc sql noprint; -create table &tempds1 as - select distinct &comma_pk,&retained_key - from &base_libds &filter_str - order by &comma_pk,&retained_key; - -%if &check_uniqueness=YES %then %do; - select count(*) into:checknobs - from (select distinct &comma_pk from &app_libds); - select count(*) into: appnobs from &app_libds; /* might be view */ - %if &checknobs ne &appnobs %then %do; - %let msg=Source table &app_libds is not unique on (&business_key); - %let iserr=1; - %end; -%end; -%if &iserr=1 %then %do; - /* err case so first perform an unlock of the base table before exiting */ - %mp_lockanytable( - UNLOCK,lib=&base_lib,ds=&base_dsn,ref=%superq(msg),ctl_ds=&locktable - ) -%end; -%mp_abort(iftrue= (&iserr=1),mac=mp_retainedkey,msg=%superq(msg)) - -%if %mf_existvar(&app_libds,&retained_key) -%then %let dropvar=(drop=&retained_key); - -/* prepare interim table with retained key populated for matching keys */ -proc sql noprint; -create table &tempds2 as - select b.&retained_key, a.* - from &app_libds &dropvar a - left join &tempds1 b - on 1 - %do idx_pk=1 %to %sysfunc(countw(&business_key)); - %let idx_val=%scan(&business_key,&idx_pk); - and a.&idx_val=b.&idx_val - %end; - order by &retained_key; - -/* identify the number of entries without retained keys (new records) */ -select count(*) into: newkey_cnt - from &tempds2 - where missing(&retained_key); -quit; - -/** - * Update maxkey table if link provided - */ -%if &maxkeytable ne 0 %then %do; - proc sql noprint; - select count(*) into: check from &maxkeytable - where upcase(keytable)="&base_libds"; - - %mp_lockanytable(LOCK - ,lib=%scan(&maxkeytable,1,.) - ,ds=%scan(&maxkeytable,2,.) - ,ref=Updating maxkeyvalues with mp_retainedkey - ,ctl_ds=&locktable - ) - proc sql; - %if &check=0 %then %do; - insert into &maxkeytable - set keytable="&base_libds" - ,keycolumn="&retained_key" - ,max_key=%eval(&maxkey+&newkey_cnt) - ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt; - %end; - %else %do; - update &maxkeytable - set max_key=%eval(&maxkey+&newkey_cnt) - ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt - where keytable="&base_libds"; - %end; - %mp_lockanytable(UNLOCK - ,lib=%scan(&maxkeytable,1,.) - ,ds=%scan(&maxkeytable,2,.) - ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt) - ,ctl_ds=&locktable - ) -%end; - -/* fill in the missing retained key values */ -%let tempvar=%mf_getuniquename(); -data &outds(drop=&tempvar); - retain &tempvar %eval(&maxkey+1); - set &tempds2; - if &retained_key =. then &retained_key=&tempvar; - &tempvar=&tempvar+1; -run; - -%mend mp_retainedkey; - diff --git a/009_macros/mp_runddl.sas b/009_macros/mp_runddl.sas deleted file mode 100644 index 0287b90..0000000 --- a/009_macros/mp_runddl.sas +++ /dev/null @@ -1,49 +0,0 @@ -/*** HELP START ***//** - @file mp_runddl.sas - @brief An opinionated way to execute DDL files in SAS. - @details When delivering projects there should be seperation between the DDL - used to generate the tables and the sample data used to populate them. - - This macro expects certain folder structure - eg: - - rootlib - |-- LIBREF1 - | |__ mytable.ddl - | |__ someothertable.ddl - |-- LIBREF2 - | |__ table1.ddl - | |__ table2.ddl - |-- LIBREF3 - |__ table3.ddl - |__ table4.ddl - - Only files with the .ddl suffix are executed. The parent folder name is used - as the libref. - Files should NOT contain the `proc sql` statement - this is to prevent - statements being executed if there is an err condition. - - Usage: - - %mp_runddl(/some/rootlib) * execute all libs ; - - %mp_runddl(/some/rootlib, inc=LIBREF1 LIBREF2) * include only these libs; - - %mp_runddl(/some/rootlib, exc=LIBREF3) * same as above ; - - - @param [in] path location of the DDL folder structure - @param [in] inc= list of librefs to include - @param [in] exc= list of librefs to exclude (takes precedence over inc=) - - @version 9.3 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_runddl(path, inc=, exc= -)/*/STORE SOURCE*/; - - - -%mend mp_runddl; diff --git a/009_macros/mp_searchcols.sas b/009_macros/mp_searchcols.sas deleted file mode 100644 index 26e35ad..0000000 --- a/009_macros/mp_searchcols.sas +++ /dev/null @@ -1,104 +0,0 @@ -/*** HELP START ***//** - @file mp_searchcols.sas - @brief Searches all columns in a library - @details - Scans a set of libraries and creates a dataset containing all source tables - containing one or more of a particular set of columns - - Usage: - - %mp_searchcols(libs=sashelp work, cols=name sex age) - - @param [in] libs= (SASHELP) - Space separated list of libraries to search for columns - @param [in] cols= - Space separated list of column names to search for (not case sensitive) - @param [out] outds= (mp_searchcols) - The table to create with the results. Will have one line per table match. - @param [in] match= (ANY) The match type. Valid values: - @li ANY - The table contains at least one of the columns - @li WILD - The table contains a column with a name that partially matches - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_searchcols(libs=sashelp - ,cols= - ,outds=mp_searchcols - ,match=ANY -)/*/STORE SOURCE*/; - -%put &sysmacroname process began at %sysfunc(datetime(),datetime19.); - -/* get the list of tables in the library */ -proc sql; -create table _data_ as - select distinct upcase(libname) as libname - , upcase(memname) as memname - , upcase(name) as name - from dictionary.columns -%if %sysevalf(%superq(libs)=,boolean)=0 %then %do; - where upcase(libname) in ("IMPOSSIBLE", - %local x; - %do x=1 %to %sysfunc(countw(&libs)); - "%upcase(%scan(&libs,&x))" - %end; - ) -%end; - order by 1,2,3; - -%local tempds; -%let tempds=&syslast; -data &outds; - set &tempds; - length cols matchcols $32767; - cols=upcase(symget('cols')); - colcount=countw(cols); - by libname memname name; - if _n_=1 then do; - putlog "Searching libs: &libs"; - putlog "Searching cols: " cols; - end; - if first.memname then do; - sumcols=0; - retain matchcols; - matchcols=''; - end; -%if &match=ANY %then %do; - if findw(cols,name,,'spit') then do; - sumcols+1; - matchcols=cats(matchcols)!!' '!!cats(name); - end; -%end; -%else %if &match=WILD %then %do; - if _n_=1 then do; - retain wcount; - wcount=countw(cols); - drop wcount; - end; - do i=1 to wcount; - length curword $32; - curword=scan(cols,i,' '); - drop curword; - if index(name,cats(curword)) then do; - sumcols+1; - matchcols=cats(matchcols)!!' '!!cats(curword); - end; - end; -%end; - - if last.memname then do; - if sumcols>0 then output; - if sumcols=colcount then putlog "Full Match: " libname memname; - end; - keep libname memname sumcols matchcols; -run; - -proc sort; by descending sumcols memname libname; run; - -proc sql; -drop table &tempds; -%put &sysmacroname process finished at %sysfunc(datetime(),datetime19.); - -%mend mp_searchcols; diff --git a/009_macros/mp_searchdata.sas b/009_macros/mp_searchdata.sas deleted file mode 100644 index c795de2..0000000 --- a/009_macros/mp_searchdata.sas +++ /dev/null @@ -1,131 +0,0 @@ -/*** HELP START ***//** - @file - @brief Searches all data in a library - @details - Scans an entire library and creates a copy of any table - containing a specific string OR numeric value. Only - matching records are written out. - If both a string and numval are provided, the string - will take precedence. - - Usage: - - %mp_searchdata(lib=sashelp, string=Jan) - %mp_searchdata(lib=sashelp, ds=bird, numval=1) - %mp_searchdata(lib=sashelp, ds=class, string=l,outobs=5) - - - Outputs zero or more tables to an MPSEARCH library with specific records. - - @param [in] lib= The libref to search (should be already assigned) - @param [in] ds= The dataset to search (leave blank to search entire library) - @param [in] string= String value to search (case sensitive, can be partial) - @param [in] numval= Numeric value to search (must be exact) - @param [out] outloc= (0) Optionally specify the directory in which to - create the the output datasets with matching rows. By default it will - write them to a temporary subdirectory within the WORK folder. - @param [out] outlib= (MPSEARCH) Assign a different libref to the output - library containing the matching datasets / records - @param [in] outobs= set to a positive integer to restrict the number of - observations - @param [in] filter_text= (1=1) Add a (valid) filter clause to further filter - the results. - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_getvarlist.sas - @li mf_getvartype.sas - @li mf_mkdir.sas - @li mf_nobs.sas - - @version 9.2 - @author Allan Bowe -**//*** HELP END ***/ - -%macro mp_searchdata(lib= - ,ds= - ,string= /* the query will use a contains (?) operator */ - ,numval= /* numeric must match exactly */ - ,outloc=0 - ,outlib=MPSEARCH - ,outobs=-1 - ,filter_text=%str(1=1) -)/*/STORE SOURCE*/; - -%local table_list table table_num table colnum col start_tm check_tm vars type - coltype; -%put process began at %sysfunc(datetime(),datetime19.); - -%if &syscc ge 4 %then %do; - %put %str(WAR)NING: SYSCC=&syscc on macro entry; - %return; -%end; - -%if &string = %then %let type=N; -%else %let type=C; - -%if "&outloc"="0" %then %do; - %let outloc=%sysfunc(pathname(work))/%mf_getuniquename(); -%end; - -%mf_mkdir(&outloc) -libname &outlib "&outloc"; - -/* get the list of tables in the library */ -proc sql noprint; -select distinct memname into: table_list separated by ' ' - from dictionary.tables - where upcase(libname)="%upcase(&lib)" -%if &ds ne %then %do; - and upcase(memname)=%upcase("&ds") -%end; - ; -/* check that we have something to check */ -%if %length(&table_list)=0 %then %put library &lib contains no tables!; -/* loop through each table */ -%else %do table_num=1 %to %sysfunc(countw(&table_list,%str( ))); - %let table=%scan(&table_list,&table_num,%str( )); - %let vars=%mf_getvarlist(&lib..&table); - %if %length(&vars)=0 %then %do; - %put NO COLUMNS IN &lib..&table! This will be skipped.; - %end; - %else %do; - %let check_tm=%sysfunc(datetime()); - /* prep input */ - data &outlib..&table; - set &lib..&table; - where %unquote(&filter_text) and ( 0 - /* loop through columns */ - %do colnum=1 %to %sysfunc(countw(&vars,%str( ))); - %let col=%scan(&vars,&colnum,%str( )); - %let coltype=%mf_getvartype(&lib..&table,&col); - %if &type=C and &coltype=C %then %do; - /* if a char column, see if it contains the string */ - or ("&col"n ? "&string") - %end; - %else %if &type=N and &coltype=N %then %do; - /* if numeric match exactly */ - or ("&col"n = &numval) - %end; - %end; - ); - %if &outobs>-1 %then %do; - if _n_ > &outobs then stop; - %end; - run; - %put Search query for &table took - %sysevalf(%sysfunc(datetime())-&check_tm) seconds; - %if &syscc ne 0 %then %do; - %put %str(ERR)ROR: SYSCC=&syscc when processing &lib..&table; - %return; - %end; - %if %mf_nobs(&outlib..&table)=0 %then %do; - proc sql; - drop table &outlib..&table; - %end; - %end; -%end; - -%put process finished at %sysfunc(datetime(),datetime19.); - -%mend mp_searchdata; diff --git a/009_macros/mp_setkeyvalue.sas b/009_macros/mp_setkeyvalue.sas deleted file mode 100644 index 197c6ec..0000000 --- a/009_macros/mp_setkeyvalue.sas +++ /dev/null @@ -1,55 +0,0 @@ -/*** HELP START ***//** - @file - @brief Logs a key value pair a control dataset - @details If the dataset does not exist, it is created. Usage: - - %mp_setkeyvalue(someindex,22,type=N) - %mp_setkeyvalue(somenewindex,somevalue) - -

SAS Macros

- @li mf_existds.sas - -

Related Macros

- @li mf_getvalue.sas - - @param [in] key Provide a key on which to perform the lookup - @param [in] value Provide a value - @param [in] type= either C or N will populate valc and valn respectively. - C is default. - @param [out] libds= define the target table to hold the parameters - - @version 9.2 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_setkeyvalue(key,value,type=C,libds=work.mp_setkeyvalue -)/*/STORE SOURCE*/; - - %if not (%mf_existds(&libds)) %then %do; - data &libds (index=(key/unique)); - length key $64 valc $2048 valn 8 type $1; - call missing(of _all_); - stop; - run; - %end; - - proc sql; - delete from &libds - where key=symget('key'); - insert into &libds - set key=symget('key') - %if &type=C %then %do; - ,valc=symget('value') - ,type='C' - %end; - %else %do; - ,valn=symgetn('value') - ,type='N' - %end; - ; - - quit; - -%mend mp_setkeyvalue; diff --git a/009_macros/mp_sortinplace.sas b/009_macros/mp_sortinplace.sas deleted file mode 100644 index 622be12..0000000 --- a/009_macros/mp_sortinplace.sas +++ /dev/null @@ -1,122 +0,0 @@ -/*** HELP START ***//** - @file - @brief Sorts a SAS dataset in place, preserving constraints - @details Generally if a dataset contains indexes, then it is not necessary to - sort it before performing operations such as merges / joins etc. - That said, there are a few edge cases where it can be desirable: - - @li To allow adjacent records to be viewed directly in the dataset - @li To apply compression, or to remove deleted records - @li To improve performance for specific queries - - This macro will only work for BASE (V9) engine libraries. It works by - creating a copy of the dataset (without data, WITH constraints) in the same - library, appending a sorted view into it, and finally - renaming it. - - Example usage: - - proc sql; - create table work.example as - select * from sashelp.class; - alter table work.example - add constraint pk primary key(name); - %mp_sortinplace(work.example) - - @param [in] libds The libref.datasetname that needs to be sorted - -

SAS Macros

- @li mf_existds.sas - @li mf_getengine.sas - @li mf_getquotedstr.sas - @li mf_getuniquename.sas - @li mf_getvarlist.sas - @li mf_nobs.sas - @li mp_abort.sas - @li mp_getpk.sas - -

Related Macros

- @li mp_sortinplace.test.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_sortinplace(libds -)/*/STORE SOURCE*/; - -%local lib ds tempds1 tempds2 tempvw sortkey; - -/* perform validations */ -%mp_abort(iftrue=(%sysfunc(countc(&libds,.)) ne 1) - ,mac=mp_sortinplace - ,msg=%str(LIBDS (&libds) should have LIBREF.DATASET format) -) -%mp_abort(iftrue=(%mf_existds(&libds)=0) - ,mac=mp_sortinplace - ,msg=%str(&libds does not exist) -) - -%let lib=%scan(&libds,1,.); -%let ds=%scan(&libds,2,.); -%mp_abort(iftrue=(%mf_getengine(&lib) ne V9) - ,mac=mp_sortinplace - ,msg=%str(&lib is not a BASE engine library) -) - -/* grab a copy of the constraints so we know what to sort by */ -%let tempds1=%mf_getuniquename(prefix=&sysmacroname); -%mp_getpk(lib=&lib,ds=&ds,outds=work.&tempds1) - -%if %mf_nobs(work.&tempds1)=0 %then %do; - %put &sysmacroname: No PK found in &lib..&ds; - %put Sorting will not take place; - %return; -%end; - -/* fallback sortkey is all fields */ -%let sortkey=%mf_getvarlist(&libds); - -/* overlay actual sort key if it exists */ -data _null_; - set work.&tempds1; - call symputx('sortkey',coalescec(pk_fields,symget('sortkey'))); -run; - - -/* create empty copy, with ALL constraints, in the same library */ -%let tempds2=%mf_getuniquename(prefix=&sysmacroname); -proc append base=&lib..&tempds2 data=&libds(obs=0); -run; - -/* create sorted view */ -%let tempvw=%mf_getuniquename(prefix=&sysmacroname); -proc sql; -create view work.&tempvw as select * from &lib..&ds -order by %mf_getquotedstr(&sortkey,quote=N); - -/* append sorted data */ -proc append base=&lib..&tempds2 data=work.&tempvw; -run; - -/* do validations */ -%mp_abort(iftrue=(&syscc ne 0) - ,mac=mp_sortinplace - ,msg=%str(syscc=&syscc prior to replace operation) -) -%mp_abort(iftrue=(%mf_nobs(&lib..&tempds2) ne %mf_nobs(&lib..&ds)) - ,mac=mp_sortinplace - ,msg=%str(new dataset has a different number of logical obs to the old) -) - -/* drop old dataset */ -proc sql; -drop table &lib..&ds; - -/* rename the new dataset */ -proc datasets library=&lib; - change &tempds2=&ds; -run; - - -%mend mp_sortinplace; diff --git a/009_macros/mp_stprequests.sas b/009_macros/mp_stprequests.sas deleted file mode 100644 index 9957779..0000000 --- a/009_macros/mp_stprequests.sas +++ /dev/null @@ -1,75 +0,0 @@ -/*** HELP START ***//** - @file - @brief Capture session start / finish times and request details - @details For details, see - https://rawsas.com/event-logging-of-stored-process-server-sessions. - Requires a base table in the following structure (name can be changed): - - proc sql; - create table &libds( - request_dttm num not null format=datetime. - ,status_cd char(4) not null - ,_metaperson varchar(100) not null - ,_program varchar(500) - ,sysuserid varchar(50) - ,sysjobid varchar(12) - ,_sessionid varchar(50) - ); - - Called via STP init / term events (configurable in SMC) as follows: - - %mp_stprequests(status_cd=INIT, libds=YOURLIB.DATASET ) - - - @param [in] status_cd= Use INIT for INIT and TERM for TERM events - @param [in] libds= (somelib.stp_requests) Location of base table - (library.dataset). To minimise risk of table locks, we HIGHLY recommend - using a database (NOT a SAS dataset). - THE LIBRARY SHOULD BE ASSIGNED ALREADY - eg in autoexec or earlier in the - init program proper. - - @version 9.2 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_stprequests(status_cd= /* $4 eg INIT or TERM */ - ,libds=somelib.stp_requests /* base table location */ -)/*/STORE SOURCE*/; - - /* set nosyntaxcheck so the code runs regardless */ - %local etls_syntaxcheck; - %let etls_syntaxcheck=%sysfunc(getoption(syntaxcheck)); - options nosyntaxcheck; - - data ; - if 0 then set &libds; - request_dttm=datetime(); - status_cd="&status_cd"; - _METAPERSON="&_metaperson"; - _PROGRAM="&_program"; - SYSUSERID="&sysuserid"; - SYSJOBID="&sysjobid"; - %if not %symexist(_SESSIONID) %then %do; - /* session id is stored in the replay variable but needs to be extracted */ - _replay=symget('_replay'); - _replay=subpad(_replay,index(_replay,'_sessionid=')+11,length(_replay)); - index=index(_replay,'&')-1; - if index=-1 then index=length(_replay); - _replay=substr(_replay,1,index); - _SESSIONID=_replay; - drop _replay index; - %end; - %else %do; - /* explicitly created sessions are automatically available */ - _SESSIONID=symget('_SESSIONID'); - %end; - output; - stop; - run; - - proc append base=&libds data=&syslast nowarn;run; - - options &etls_syntaxcheck; -%mend mp_stprequests; diff --git a/009_macros/mp_streamfile.sas b/009_macros/mp_streamfile.sas deleted file mode 100644 index f2ff172..0000000 --- a/009_macros/mp_streamfile.sas +++ /dev/null @@ -1,212 +0,0 @@ -/*** HELP START ***//** - @file - @brief Streams a file to _webout according to content type - @details Will set headers using appropriate functions per the server type - (Viya, EBI, [SASjs Server](https://github.com/sasjs/server)) and stream - content using mp_binarycopy(). - - Usage: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt) - - @param [in] contenttype= (TEXT) Supported: - @li CSV - @li EXCEL - @li MARKDOWN - @li TEXT - @li ZIP - Feel free to submit PRs to support more mime types! The official list is - here: https://www.iana.org/assignments/media-types/media-types.xhtml - @param [in] inloc= /path/to/file.ext to be sent - @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`) - @param [in] iftrue= (1=1) Provide a condition under which to execute. - @param [out] outname= the name of the file, as downloaded by the browser - @param [out] outref= (_webout) The destination where the file should be - streamed. - -

SAS Macros

- @li mf_getplatform.sas - @li mfs_httpheader.sas - @li mp_binarycopy.sas - - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_streamfile( - contenttype=TEXT - ,inloc= - ,inref=0 - ,iftrue=%str(1=1) - ,outname= - ,outref=_webout -)/*/STORE SOURCE*/; - -%if not(%eval(%unquote(&iftrue))) %then %return; - -%let contentype=%upcase(&contenttype); -%let outref=%upcase(&outref); -%local platform; %let platform=%mf_getplatform(); - -/** - * check engine type to avoid the below err message: - * > Function is only valid for filerefs using the CACHE access method. - */ -%local streamweb; -%let streamweb=0; -data _null_; - set sashelp.vextfl(where=(upcase(fileref)="&outref")); - if xengine='STREAM' then call symputx('streamweb',1,'l'); -run; - -%if &contentype=CSV %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type','application/csv'); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt' - contenttype='application/csv' - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,application/csv) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %if &contentype=EXCEL %then %do; - /* suitable for XLS format */ - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type','application/vnd.ms-excel'); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls' - contenttype='application/vnd.ms-excel' - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,application/vnd.ms-excel) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type',"image/%lowcase(&contenttype)"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" - contenttype="image/%lowcase(&contenttype)"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,image/%lowcase(&contenttype)) - %end; -%end; -%else %if &contentype=HTML or &contenttype=MARKDOWN %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type',"text/%lowcase(&contenttype)"); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json" - contenttype="text/%lowcase(&contenttype)" - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,text/%lowcase(&contenttype)) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %if &contentype=TEXT %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type','application/text'); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.txt' - contenttype='application/text' - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,application/text) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type',"font/%lowcase(&contenttype)"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" - contenttype="font/%lowcase(&contenttype)"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,font/%lowcase(&contenttype)) - %end; -%end; -%else %if &contentype=XLSX %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type', - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls' - contenttype= - 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type - ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet - ) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %if &contentype=ZIP %then %do; - %if (&platform=SASMETA and &streamweb=1) %then %do; - data _null_; - rc=stpsrv_header('Content-Type','application/zip'); - rc=stpsrv_header('Content-disposition',"attachment; filename=&outname"); - run; - %end; - %else %if &platform=SASVIYA %then %do; - filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip' - contenttype='application/zip' - contentdisp="attachment; filename=&outname"; - %end; - %else %if &platform=SASJS %then %do; - %mfs_httpheader(Content-Type,application/zip) - %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname)) - %end; -%end; -%else %do; - %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!; -%end; - -%if &inref ne 0 %then %do; - %mp_binarycopy(inref=&inref,outref=&outref) -%end; -%else %do; - %mp_binarycopy(inloc="&inloc",outref=&outref) -%end; - -%mend mp_streamfile; diff --git a/009_macros/mp_testjob.sas b/009_macros/mp_testjob.sas deleted file mode 100644 index f5610b0..0000000 --- a/009_macros/mp_testjob.sas +++ /dev/null @@ -1,92 +0,0 @@ -/*** HELP START ***//** - @file - @brief Runs arbitrary code for a specified amount of time - @details Executes a series of procs and data steps to enable performance - testing of arbitrary jobs. - - %mp_testjob( - duration=60*5 - ) - - @param [in] duration= (30) The time in seconds which the job should run for. - Actual time may vary, as the check is done in between steps. - -

SAS Macros

- @li mf_getuniquelibref.sas - @li mf_getuniquename.sas - @li mf_mkdir.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_testjob(duration=30 -)/*/STORE SOURCE*/; -%local lib dir ds1 ds2 ds3 start_tm i; - -%let start_tm=%sysfunc(datetime()); -%let duration=%sysevalf(&duration); - -/* create a temporary library in WORK */ -%let lib=%mf_getuniquelibref(); -%let dir=%mf_getuniquename(); -%mf_mkdir(%sysfunc(pathname(work))/&dir) -libname &lib "%sysfunc(pathname(work))/&dir"; - -/* loop through until time expires */ -%let ds1=%mf_getuniquename(); -%let ds2=%mf_getuniquename(); -%let ds3=%mf_getuniquename(); -%do i=0 %to 1; - - /* create big dataset */ - data &lib..&ds1(compress=no ); - do x=1 to 1000000; - randnum0=ranuni(0)*3; - randnum1=ranuni(0)*2; - bigchar=repeat('A',300); - output; - end; - run; - %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate; - - proc summary ; - class randnum0 randnum1; - output out=&lib..&ds2; - run;quit; - %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate; - - /* add more data */ - proc sql; - create table &lib..&ds3 as - select *, ranuni(0)*10 as randnum2 - from &lib..&ds1 - order by randnum1; - quit; - %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate; - - proc sort data=&lib..&ds3; - by descending x; - run; - %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate; - - /* wait 5 seconds */ - data _null_; - call sleep(5,1); - run; - %if %sysevalf( (%sysfunc(datetime())-&start_tm)>&duration ) %then %goto gate; - - %let i=0; - -%end; - -%gate: -%put time is up!; -proc datasets lib=&lib kill; -run; -quit; -libname &lib clear; - - -%mend mp_testjob; diff --git a/009_macros/mp_testservice.sas b/009_macros/mp_testservice.sas deleted file mode 100644 index 81a157a..0000000 --- a/009_macros/mp_testservice.sas +++ /dev/null @@ -1,39 +0,0 @@ -/*** HELP START ***//** - @file - @brief To be deprecated. Will execute a SASjs web service on SAS 9 or Viya - @details Use the mx_testservice.sas macro instead (documentation can be - found there) - -

SAS Macros

- @li mx_testservice.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_testservice(program, - inputfiles=0, - inputdatasets=0, - inputparams=0, - debug=log, - mdebug=0, - outlib=0, - outref=0, - viyaresult=WEBOUT_JSON, - viyacontext=SAS Job Execution compute context -)/*/STORE SOURCE*/; - -%mx_testservice(&program, - inputfiles=&inputfiles, - inputdatasets=&inputdatasets, - inputparams=&inputparams, - debug=&debug, - mdebug=&mdebug, - outlib=&outlib, - outref=&outref, - viyaresult=&viyaresult, - viyacontext=&viyacontext -) - -%mend mp_testservice; diff --git a/009_macros/mp_testwritespeedlibrary.sas b/009_macros/mp_testwritespeedlibrary.sas deleted file mode 100644 index 468573f..0000000 --- a/009_macros/mp_testwritespeedlibrary.sas +++ /dev/null @@ -1,59 +0,0 @@ -/*** HELP START ***//** - @file mp_testwritespeedlibrary.sas - @brief Tests the write speed of a new table in a SAS library - @details Will create a new table of a certain size in an - existing SAS library. The table will have one column, - and will be subsequently deleted. - - %mp_testwritespeedlibrary( - lib=work - ,size=0.5 - ,outds=work.results - ) - - @param [in] lib= (WORK) The library in which to create the table - @param [in] size= (0.1) The size in GB of the table to create - @param [out] outds= (WORK.RESULTS) The output dataset to be created. - -

SAS Macros

- @li mf_getuniquename.sas - @li mf_existds.sas - - @version 9.4 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_testwritespeedlibrary(lib=WORK - ,outds=work.results - ,size=0.1 -)/*/STORE SOURCE*/; -%local ds start; - -/* find an unused, unique name for the new table */ -%let ds=%mf_getuniquename(); -%do %until(%mf_existds(&lib..&ds)=0); - %let ds=%mf_getuniquename(); -%end; - -%let start=%sysfunc(datetime()); - -data &lib..&ds(compress=no keep=x); - header=128*1024; - size=(1073741824/8 * &size) - header; - do x=1 to size; - output; - end; -run; - -proc sql; -drop table &lib..&ds; - -data &outds; - lib="&lib"; - start_dttm=put(&start,datetime19.); - end_dttm=put(datetime(),datetime19.); - duration_seconds=end_dttm-start_dttm; -run; - -%mend mp_testwritespeedlibrary; diff --git a/009_macros/mp_tree.sas b/009_macros/mp_tree.sas deleted file mode 100644 index 467e8ff..0000000 --- a/009_macros/mp_tree.sas +++ /dev/null @@ -1,71 +0,0 @@ -/*** HELP START ***//** - @file - @brief Recursively scans a directory tree to get all subfolders and content - @details - Usage: - - %mp_tree(dir=/tmp, outds=work.tree) - - Credits: - - Roger Deangelis: -https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887 - - Tom: -https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419 - - - @param [in] dir= (/tmp) Directory to be scanned - @param [out] outds= (work.mp_tree) Dataset to create - - @returns outds contains the following variables: - - - `dir`: a flag (1/0) to say whether it is a directory or not. This is not - reliable - folders that you do not have permission to open will be flagged - as directories. - - `ext`: file extension - - `filename`: file name - - `dirname`: directory name - - `fullpath`: directory + file name - - @version 9.2 -**//*** HELP END ***/ - -%macro mp_tree(dir=/tmp - ,outds=work.mp_tree -)/*/STORE SOURCE*/; - -data &outds ; - length dir 8 ext filename dirname $256 fullpath $512 ; - call missing(of _all_); - fullpath = "&dir"; -run; - -%local sep; -%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\; -%else %let sep=/; - -data &outds ; - modify &outds ; - retain sep "&sep"; - rc=filename('tmp',fullpath); - dir_id=dopen('tmp'); - dir = (dir_id ne 0) ; - if dir then dirname=fullpath; - else do; - filename=scan(fullpath,-1,sep) ; - dirname =substrn(fullpath,1,length(fullpath)-length(filename)); - if index(filename,'.')>1 then ext=scan(filename,-1,'.'); - end; - replace; - if dir then do; - do i=1 to dnum(dir_id); - fullpath=cats(dirname,sep,dread(dir_id,i)); - output; - end; - rc=dclose(dir_id); - end; - rc=filename('tmp'); -run; - -%mend mp_tree; diff --git a/009_macros/mp_unzip.sas b/009_macros/mp_unzip.sas deleted file mode 100644 index 0e81659..0000000 --- a/009_macros/mp_unzip.sas +++ /dev/null @@ -1,91 +0,0 @@ -/*** HELP START ***//** - @file mp_unzip.sas - @brief Unzips a zip file - @details Opens the zip file and copies all the contents to another directory. - It is not possible to retain permissions / timestamps, also the BOF marker - is lost so it cannot extract binary files. - - Usage: - - filename mc url - "https://raw.githubusercontent.com/sasjs/core/main/all.sas"; - %inc mc; - - %mp_unzip(ziploc="/some/file.zip",outdir=/some/folder) - - More info: - https://blogs.sas.com/content/sasdummy/2015/05/11/using-filename-zip-to-unzip-and-read-data-files-in-sas/ - - @param [in] ziploc= Fileref or quoted full path, eg: "/path/to/file.zip" - @param [out] outdir= (%sysfunc(pathname(work))) Directory in which to write - the outputs (created if needed) - -

SAS Macros

- @li mf_mkdir.sas - @li mf_getuniquefileref.sas - @li mp_binarycopy.sas - - @version 9.4 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_unzip( - ziploc= - ,outdir=%sysfunc(pathname(work)) -)/*/STORE SOURCE*/; - -%local f1 f2 ; -%let f1=%mf_getuniquefileref(); -%let f2=%mf_getuniquefileref(); - -/* Macro variable &datazip would be read from the file */ -filename &f1 ZIP &ziploc; - -/* create target folder */ -%mf_mkdir(&outdir) - -/* Read the "members" (files) from the ZIP file */ -data _data_(keep=memname isFolder); - length memname $200 isFolder 8; - fid=dopen("&f1"); - if fid=0 then stop; - memcount=dnum(fid); - do i=1 to memcount; - memname=dread(fid,i); - /* check for trailing / in folder name */ - isFolder = (first(reverse(trim(memname)))='/'); - output; - end; - rc=dclose(fid); -run; - -filename &f2 temp; - -/* loop through each entry and either create the subfolder or extract member */ -data _null_; - set &syslast; - file &f2; - if isFolder then call execute('%mf_mkdir(&outdir/'!!memname!!')'); - else do; - qname=quote(cats("&outdir/",memname)); - bname=cats('(',memname,')'); - put '/* hat tip: "data _null_" on SAS-L */'; - put 'data _null_;'; - put ' infile &f1 ' bname ' lrecl=256 recfm=F length=length eof=eof unbuf;'; - put ' file ' qname ' lrecl=256 recfm=N;'; - put ' input;'; - put ' put _infile_ $varying256. length;'; - put ' return;'; - put 'eof:'; - put ' stop;'; - put 'run;'; - end; -run; - -%inc &f2/source2; - -filename &f2 clear; - -%mend mp_unzip; diff --git a/009_macros/mp_updatevarlength.sas b/009_macros/mp_updatevarlength.sas deleted file mode 100644 index 22d2e0c..0000000 --- a/009_macros/mp_updatevarlength.sas +++ /dev/null @@ -1,93 +0,0 @@ -/*** HELP START ***//** - @file mp_updatevarlength.sas - @brief Change the length of a variable - @details The library is assumed to be assigned. Simple character updates - currently supported, numerics are more complicated and will follow. - - data example; - a='1'; - b='12'; - c='123'; - run; - %mp_updatevarlength(example,a,3) - %mp_updatevarlength(example,c,1) - proc sql; - describe table example; - - @param [in] libds the library.dataset to be modified - @param [in] var The variable to modify - @param [in] len The new length to apply - -

SAS Macros

- @li mf_existds.sas - @li mp_abort.sas - @li mf_existvar.sas - @li mf_getvarlen.sas - @li mf_getvartype.sas - @li mf_getnobs.sas - @li mp_createconstraints.sas - @li mp_getconstraints.sas - @li mp_deleteconstraints.sas - - @version 9.2 - @author Allan Bowe - -**//*** HELP END ***/ - -%macro mp_updatevarlength(libds,var,len -)/*/STORE SOURCE*/; - -%if %index(&libds,.)=0 %then %let libds=WORK.&libds; - -%mp_abort(iftrue=(%mf_existds(&libds)=0) - ,mac=&sysmacroname - ,msg=%str(Table &libds not found!) -) - -%mp_abort(iftrue=(%mf_existvar(&libds,&var)=0) - ,mac=&sysmacroname - ,msg=%str(Variable &var not found on &libds!) -) - -/* not possible to in-place modify a numeric length, to add later */ -%mp_abort(iftrue=(%mf_getvartype(&libds,&var)=0) - ,mac=&sysmacroname - ,msg=%str(Only character resizings are currently supported) -) - -%local oldlen; -%let oldlen=%mf_getvarlen(&libds,&var); -%if &oldlen=&len %then %do; - %put &sysmacroname: Old and new lengths (&len) match!; - %return; -%end; - -%let libds=%upcase(&libds); - - -data;run; -%local dsconst; %let dsconst=&syslast; -%mp_getconstraints(lib=%scan(&libds,1,.),ds=%scan(&libds,2,.),outds=&dsconst) - -%mp_abort(iftrue=(&syscc ne 0) - ,mac=&sysmacroname - ,msg=%str(syscc=&syscc) -) - -%if %mf_getnobs(&dscont)=0 %then %do; - /* must use SQL as proc datasets does not support length changes */ - proc sql; - alter table &libds modify &var char(&len); - %return; -%end; - -/* we have constraints! */ - -%mp_deleteconstraints(inds=&dsconst,outds=&dsconst._dropd,execute=YES) - -proc sql; -alter table &libds modify &var char(&len); - -%mp_createconstraints(inds=&dsconst,outds=&dsconst._addd,execute=YES) - -%mend mp_updatevarlength; diff --git a/009_macros/mp_wait4file.sas b/009_macros/mp_wait4file.sas deleted file mode 100644 index f52dfe9..0000000 --- a/009_macros/mp_wait4file.sas +++ /dev/null @@ -1,35 +0,0 @@ -/*** HELP START ***//** - @file - @brief Wait until a file arrives before continuing execution - @details Loops with a `sleep()` command until a file arrives or the max wait - period expires. - - Example: Wait 3 minutes OR for /tmp/flag.txt to appear - - %mp_wait4file(/tmp/flag.txt , maxwait=60*3) - - @param [in] file The file to wait for. Must be provided. - @param [in] maxwait= (0) Number of seconds to wait. If set to zero, will - loop indefinitely (to a maximum of 46 days, per SAS [documentation]( - https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a001418809.htm - )). Otherwise, execution will proceed upon sleep expiry. - @param [in] interval= (1) The wait period between sleeps, in seconds - - -**//*** HELP END ***/ - -%macro mp_wait4file(file, maxwait=0, interval=1); - -%if %str(&file)=%str() %then %do; - %put %str(ERR)OR: file not provided; -%end; - -data _null_; - maxwait=&maxwait; - if maxwait=0 then maxwait=60*60*24*46; - do until (fileexist("&file") or slept>maxwait ); - slept=sum(slept,sleep(&interval,1)); - end; -run; - -%mend mp_wait4file; diff --git a/009_macros/mp_webin.sas b/009_macros/mp_webin.sas deleted file mode 100644 index 671d103..0000000 --- a/009_macros/mp_webin.sas +++ /dev/null @@ -1,59 +0,0 @@ -/*** HELP START ***//** - @file - @brief Fix the `_WEBIN` variables provided to SAS web services - @details When uploading files to SAS Stored Processes or Viya Jobs a number - of global macro variables are automatically created - however there are some - differences in behaviour both between SAS 9 and Viya, and also between a - single file upload and a multi-file upload. - - This macro "straightens" up the global macro variables to make it easier / - simpler to write code that works in both environments and with a variable - number of file inputs. - - After running this macro, the following global variables will *always* exist: - @li `_WEBIN_FILE_COUNT` - @li `_WEBIN_FILENAME1` - @li `_WEBIN_FILEREF1` - @li `_WEBIN_NAME1` - - Usage: - - %mp_webin() - - This was created as a macro procedure (over a macro function) as it will also - use the filename statement in Viya environments (where `_webin_fileuri` is - provided). - -

SAS Macros

- @li mf_getplatform.sas - @li mf_getuniquefileref.sas - -**//*** HELP END ***/ - -%macro mp_webin(); - -/* prepare global variables */ -%global _webin_file_count - _webin_filename _webin_filename1 - _webin_fileref _webin_fileref1 - _webin_fileuri _webin_fileuri1 - _webin_name _webin_name1 - ; - -/* create initial versions */ -%let _webin_file_count=%eval(&_webin_file_count+0); -%let _webin_filename1=%sysfunc(coalescec(&_webin_filename1,&_webin_filename)); -%let _webin_fileref1=%sysfunc(coalescec(&_webin_fileref1,&_webin_fileref)); -%let _webin_fileuri1=%sysfunc(coalescec(&_webin_fileuri1,&_webin_fileuri)); -%let _webin_name1=%sysfunc(coalescec(&_webin_name1,&_webin_name)); - - -/* If Viya, create temporary fileref(s) */ -%local i; -%if %mf_getplatform()=SASVIYA %then %do i=1 %to &_webin_file_count; - %let _webin_fileref&i=%mf_getuniquefileref(); - filename &&_webin_fileref&i filesrvc "&&_webin_fileuri&i"; -%end; - - -%mend mp_webin; diff --git a/009_macros/mp_zip.sas b/009_macros/mp_zip.sas deleted file mode 100644 index 76f9d76..0000000 --- a/009_macros/mp_zip.sas +++ /dev/null @@ -1,85 +0,0 @@ -/*** HELP START ***//** - @file - @brief Creates a zip file - @details For DIRECTORY usage, will ignore subfolders. For DATASET usage, - provide a column that contains the full file path to each file to be zipped. - - %mp_zip(in=myzips,type=directory,outname=myDir) - %mp_zip(in=/my/file/path.txt,type=FILE,outname=myFile) - %mp_zip(in=SOMEDS,incol=FPATH,type=DATASET,outname=myFile) - - If you are sending zipped output to the _webout destination as part of an STP - be sure that _debug is not set (else the SPWA will send non zipped content - as well). - -

SAS Macros

- @li mp_dirlist.sas - - @param [in] in= unquoted filepath, dataset of files or directory to zip - @param [in] type= (FILE) Valid values: - @li FILE - /full/path/and/filename.extension to a particular file - @li DATASET - a dataset containing a list of files to zip (see `incol`) - @li DIRECTORY - a directory to zip - @param [out] outname= (FILE) Output file to create, _without_ .zip extension - @param [out] outpath= (%sysfunc(pathname(WORK))) Parent folder for zip file - @param [in] incol= () If DATASET input, say which column contains the filepath - -

Related Macros

- @li mp_unzip.sas - @li mp_zip.test.sas - - @version 9.2 - @author Allan Bowe - @source https://github.com/sasjs/core - -**//*** HELP END ***/ - -%macro mp_zip( - in= - ,type=FILE - ,outname=FILE - ,outpath=%sysfunc(pathname(WORK)) - ,incol= - ,debug=NO -)/*/STORE SOURCE*/; - -%let type=%upcase(&type); -%local ds; - -ods package open nopf; - -%if &type=FILE %then %do; - ods package add file="&in" mimetype="application/x-compress"; -%end; -%else %if &type=DIRECTORY %then %do; - %mp_dirlist(path=&in,outds=_data_) - %let ds=&syslast; - data _null_; - set &ds; - length __command $4000; - if file_or_folder='file'; - __command=cats('ods package add file="',filepath - ,'" mimetype="application/x-compress";'); - call execute(__command); - run; - /* tidy up */ - %if &debug=NO %then %do; - proc sql; drop table &ds;quit; - %end; -%end; -%else %if &type=DATASET %then %do; - data _null_; - set ∈ - length __command $4000; - __command=cats('ods package add file="',&incol - ,'" mimetype="application/x-compress";'); - call execute(__command); - run; -%end; - - -ods package publish archive properties - (archive_name="&outname..zip" archive_path="&outpath"); -ods package close; - -%mend mp_zip; diff --git a/007_macros/mx_getgroups.sas b/009_macros/mx_getgroups.sas similarity index 100% rename from 007_macros/mx_getgroups.sas rename to 009_macros/mx_getgroups.sas diff --git a/description.sas b/description.sas index 6f91dab..f0a35c2 100644 --- a/description.sas +++ b/description.sas @@ -1,7 +1,7 @@ Type: Package Package: SASjsCore Title: SAS Macros for Application Development -Version: 4.52.2 +Version: 4.52.3 Author: Allan Bowe Maintainer: 4GL Ltd License: MIT diff --git a/sasjscore.zip b/sasjscore.zip index 79e20b7..31dea3c 100755 Binary files a/sasjscore.zip and b/sasjscore.zip differ