z/OS support: Fix compilation issues

Proper detection and flag setting

ax_check_gskssl.m4

@@ -13,18 +13,6 @@ AC_DEFUN([AX_CHECK_GSKSSL], [
               ;;
             esac
         ], [
-            # if pkg-config is installed and gskssl has installed a .pc file,
-            # then use that information and don't search ssldirs
-            AC_PATH_PROG(PKG_CONFIG, pkg-config)
-            if test x"$PKG_CONFIG" != x""; then
-                GSKSSL_LDFLAGS=`$PKG_CONFIG gskssl --libs-only-L 2>/dev/null`
-                if test $? = 0; then
-                    GSKSSL_LIBS=`$PKG_CONFIG gskssl --libs-only-l 2>/dev/null`
-                    GSKSSL_INCLUDES=`$PKG_CONFIG gskssl --cflags-only-I 2>/dev/null`
-                    found=true
-                fi
-            fi
-
             # no such luck; use some default ssldirs
             if ! $found; then
                 ssldirs="/usr/lpp/gskssl /usr"
@@ -43,7 +31,6 @@ AC_DEFUN([AX_CHECK_GSKSSL], [
             if test -f "$ssldir/include/gskssl.h"; then
                 if test -f "$ssldir/lib/GSKSSL.x"; then
                     GSKSSL_INCLUDES="-I$ssldir/include"
-                    GSKSSL_LDFLAGS="-L$ssldir/lib"
                     GSKSSL_LIBS="$ssldir/lib/GSKSSL.x $ssldir/lib/GSKCMS64.x"
                     found=true
                     AC_MSG_RESULT([yes])
@@ -62,13 +49,12 @@ AC_DEFUN([AX_CHECK_GSKSSL], [
     # being careful not to pollute the global LIBS, LDFLAGS, and CPPFLAGS
 
     AC_MSG_CHECKING([whether compiling and linking against GSK SSL works])
-    echo "Trying link with GSKSSL_LDFLAGS=$GSKSSL_LDFLAGS;" \
+    echo "Trying link with " \
         "GSKSSL_LIBS=$GSKSSL_LIBS; GSKSSL_INCLUDES=$GSKSSL_INCLUDES" >&AS_MESSAGE_LOG_FD
 
     save_LIBS="$LIBS"
     save_LDFLAGS="$LDFLAGS"
     save_CPPFLAGS="$CPPFLAGS"
-    LDFLAGS="$LDFLAGS $GSKSSL_LDFLAGS"
     LIBS="$GSKSSL_LIBS $LIBS"
     CPPFLAGS="$GSKSSL_INCLUDES $CPPFLAGS"
     AC_LINK_IFELSE(
@@ -86,5 +72,4 @@ AC_DEFUN([AX_CHECK_GSKSSL], [
 
     AC_SUBST([GSKSSL_INCLUDES])
     AC_SUBST([GSKSSL_LIBS])
-    AC_SUBST([GSKSSL_LDFLAGS])
 ])

ax_check_openssl.m4

@@ -63,7 +63,7 @@ AC_DEFUN([AX_CHECK_OPENSSL], [
 
             # no such luck; use some default ssldirs
             if ! $found; then
-                ssldirs="/usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr"
+                ssldirs="$prefix $withval /usr/local/ssl /usr/lib/ssl /usr/ssl /usr/pkg /usr/local /usr"
             fi
         ]
         )

ax_check_pkcs.m4

@@ -27,7 +27,7 @@ AC_DEFUN([AX_CHECK_PKCS], [
 
             # no such luck; use some default pkcsdirs
             if ! $found; then
-                pkcsdirs="/usr/lpp/pkcs11 /usr"
+                pkcsdirs="$withval /usr/lpp/pkcs11 /usr"
             fi
         ]
         )

configure.ac

@@ -56,21 +56,22 @@ AC_SEARCH_LIBS(dlopen, dl)
 #
 AC_HEADER_STDC
 AC_CHECK_HEADERS([CommonCrypto/CommonDigest.h])
-AC_CHECK_HEADERS([dmalloc.h direct.h errno.h fcntl.h \
-                 inttypes.h limits.h malloc.h zlib.h])
-AC_CHECK_HEADERS([sys/cdefs.h sys/file.h sys/mman.h\
-		  sys/resource.h sys/uio.h])
+AC_CHECK_HEADERS([dmalloc.h direct.h errno.h fcntl.h inttypes.h limits.h malloc.h zlib.h])
+AC_CHECK_HEADERS([sys/cdefs.h sys/file.h sys/mman.h sys/resource.h sys/uio.h])
 
 platform_name=$(uname)
+
 AS_IF([test "$platform_name" == "OS/390"],[
 AC_MSG_NOTICE([z/OS detected... Skipping some headers])
 ZLIB_FLAG="-lzz"
+NETPGP_CFLAGS="$WARNCLAGS $CFLAGS -D_XOPEN_SOURCE=600 -D_POSIX_C_SOURCE=200112L -D_XOPEN_SOURCE_EXTENDED=1 -D_UNIX03_THREADS"
 ],[
 AC_CHECK_HEADERS([sys/param.h])
 AC_CHECK_HEADERS([bzlib.h], [], [
 AC_MSG_FAILURE([missing <bzlib.h>; is bzip2 installed?])
 ])
 ZLIB_FLAG="-lz"
+NETPGP_CFLAGS="$WARNCLAGS $CFLAGS"
 ])
 AC_SUBST([ZLIB_FLAG])
 
@@ -83,17 +84,54 @@ AC_TYPE_UINT32_T
 AC_TYPE_UINT64_T
 AC_TYPE_UINT8_T
 
+found_gskssl="no"
+found_pkcs="no"
+found_openssl="no"
+
 # check for openssl
 m4_include([ax_check_openssl.m4])
 m4_include([ax_check_gskssl.m4])
 m4_include([ax_check_pkcs.m4])
 
-AX_CHECK_OPENSSL([LIBS="$LIBS $OPENSSL_LDFLAGS $OPENSSL_LIBS" INCLUDES="$INCLUDES $OPENSSL_INCLUDES" found_openssl="yes"],[found_openssl="no"])
-AX_CHECK_GSKSSL([LIBS="$LIBS $GSKSSL_LDFLAGS $GSKSSL_LIBS" INCLUDES="$INCLUDES $GSKSSL_INCLUDES" found_gskssl="yes"],[found_gskssl="no"])
-AX_CHECK_PKCS([LIBS="$LIBS $PKSC_LDFLAGS $PKCS_LIBS" INCLUDES="$INCLUDES $PKCS_INCLUDES" found_pkcs="yes"],[found_pkcs="no"])
-
-AS_IF([test "$found_openssl" == "no"],[AS_IF([test "$found_gskssl" == "no"],[AC_MSG_FAILURE([No suitable SSL library found])])])
-
+AX_CHECK_OPENSSL([found_openssl="yes"],[found_openssl="no"])
+
+AS_IF(
+    [test "$found_openssl" == "yes"],
+    [
+        LIBS="$LIBS $OPENSSL_LIBS"
+        INCLUDES="$INCLUDES $OPENSSL_INCLUDES"
+        NETPGP_CFLAGS="$OPENSSL_LDFLAGS $NETPGP_CFLAGS"
+    ],
+    [
+        AX_CHECK_GSKSSL([found_gskssl="yes"],[found_gskssl="no"])
+        AS_IF(
+            [test "$found_gskssl" == "yes"],
+            [
+                LIBS="$LIBS $GSKSSL_LIBS"
+                INCLUDES="$INCLUDES $GSKSSL_INCLUDES"
+                NETPGP_CFLAGS="$GSKSSL_LDFLAGS $NETPGP_CFLAGS"
+            ],
+            [
+                AC_MSG_FAILURE([No suitable SSL library found])
+            ]
+        )
+        AX_CHECK_PKCS([found_pkcs="yes"],[found_pkcs="no"])
+        AS_IF(
+            [test "$found_pkcs" == "yes"],
+            [
+                LIBS="$LIBS $PKCS_LIBS"
+                INCLUDES="$INCLUDES $PKCS_INCLUDES"
+                NETPGP_CFLAGS="$PKSC_LDFLAGS $NETPGP_CFLAGS"
+            ],
+            [
+                AC_MSG_FAILURE([PKCS library not found])
+            ]
+        )
+    ]
+)
+AC_SUBST([NETPGP_CFLAGS])
+
+LIBS="$LIBS $ZLIB_FLAG"
 CPPFLAGS="$CPPFLAGS $INCLUDES"
 
 AS_IF([test "$found_openssl" == "yes" ],[

include/netpgp/Makefile.am

@@ -14,5 +14,6 @@ pkginclude_HEADERS	= \
 	types.h \
 	readerwriter.h \
 	writer.h \
-	validate.h
+	validate.h \
+    netpgpdefs.h
 

src/Makefile.am

@@ -1,10 +1,6 @@
-AM_CFLAGS		= $(WARNCFLAGS)
-
 lib_LTLIBRARIES		= libnetpgp.la
-
-libnetpgp_la_CPPFLAGS	= -I$(top_srcdir)/include -I$(top_builddir)/include -D_GNU_SOURCE
-libnetpgp_la_LDFLAGS	= $(ZLIB_FLAG) 
-
+libnetpgp_la_CPPFLAGS	= $(CPPFLAGS) -I$(top_srcdir)/include -I$(top_builddir)/include -D_GNU_SOURCE
+libnetpgp_la_CFLAGS	    = $(NETPGP_CFLAGS)
 libnetpgp_la_SOURCES	= \
 	compress.c \
 	create.c \
@@ -27,9 +23,9 @@ libnetpgp_la_SOURCES	= \
     symmetric.c
 
 bin_PROGRAMS            = netpgp
-netpgp_CPPFLAGS	        = -I$(top_srcdir)/include -I$(top_builddir)/include -D_GNU_SOURCE
-LDADD                   = $(PKCS_LIBS) $(GSKSSL_LIBS) $(ZLIB_FLAG)
-netpgp_LDFLAGS	        = 
+netpgp_CPPFLAGS	        = $(CPPFLAGS) -I$(top_srcdir)/include -I$(top_builddir)/include -D_GNU_SOURCE
+netpgp_CFLAGS	        = $(NETPGP_CFLAGS)
+LDADD                   = $(LIBS)
 netpgp_SOURCES          = \
 	main.c \
 	getopt.c \