aio: change to use unix io, not stdio

This changes especially makes buffered I/O work
with non-blocking channels.

- separate read and write buffering
- support for timeout on blocking read
- read/write on same channel in event loop with buffering
- read buffer is the same across read, gets, copyto
- autoflush non-blocking writes via event loop
- copyto can now copy to any filehandle-like command
- add some copyto tests

Signed-off-by: Steve Bennett <steveb@workware.net.au>

18 files changed, 1003 insertions, 475 deletions

diff --git a/auto.def b/auto.def
index d9058bf..5531559 100644
--- a/auto.def
+++ b/auto.def
@@ -307,9 +307,8 @@ if {[get-define _FILE_OFFSET_BITS] != 64 || ![cc-check-functions stat64]} {
     cc-check-functions fstat lstat
 } else {
     # But perhaps some 32 bit systems still require explicit use of the 64 bit versions
-    cc-check-functions fstat64 lstat64
+    cc-check-functions fstat64 lstat64 lseek64
 }
-cc-check-functions fseeko ftello
 
 define TCL_LIBRARY [get-define libdir]/jim
 
diff --git a/examples/client-server.tcl b/examples/client-server.tcl
index 01b1ed2..c73849c 100644
--- a/examples/client-server.tcl
+++ b/examples/client-server.tcl
@@ -38,11 +38,9 @@ if {[os.fork] == 0} {
 	$f readable [list onread $f]
 
 	alarm 10
-	catch -signal {
-		verbose "child: in event loop"
-		vwait done
-		verbose "child: done event loop"
-	}
+	verbose "child: in event loop"
+	vwait -signal done
+	verbose "child: done event loop"
 	alarm 0
 	$f close
 	exit 0
@@ -55,9 +53,9 @@ set done 0
 set f [socket stream.server 0.0.0.0:9876]
 
 proc server_onread {f} {
-	verbose "parent: onread (server) got connection on $f"
+	verbose "parent: onread (server) got connection on [$f filename]"
 	set cfd [$f accept]
-	verbose "parent: onread accepted $cfd"
+	verbose "parent: onread accepted [$cfd filename]"
 
 	verbose "parent: read request '[string trim [$cfd gets]]'"
 
@@ -72,9 +70,7 @@ proc server_onread {f} {
 $f readable [list server_onread $f]
 
 alarm 10
-catch -signal {
-	vwait done
-}
+vwait -signal done
 alarm 0
 $f close
 
diff --git a/jim-aio.c b/jim-aio.c
index 487f2c3..0e947db 100644
--- a/jim-aio.c
+++ b/jim-aio.c
@@ -56,7 +56,6 @@
 #endif
 
 #include "jim.h"
-#include "jimiocompat.h"
 
 #if defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SELECT) && defined(HAVE_NETINET_IN_H) && defined(HAVE_NETDB_H) && defined(HAVE_ARPA_INET_H)
 #include <sys/socket.h>
@@ -70,8 +69,6 @@
 #define HAVE_SOCKETS
 #elif defined (__MINGW32__)
 /* currently mingw32 doesn't support sockets, but has pipe, fdopen */
-#else
-#define JIM_ANSIC
 #endif
 
 #if defined(JIM_SSL)
@@ -85,20 +82,23 @@
 
 #include "jim-eventloop.h"
 #include "jim-subcmd.h"
+#include "jimiocompat.h"
 
 #define AIO_CMD_LEN 32      /* e.g. aio.handleXXXXXX */
-#define AIO_BUF_LEN 256     /* Can keep this small and rely on stdio buffering */
-
-#ifndef HAVE_FTELLO
-    #define ftello ftell
-#endif
-#ifndef HAVE_FSEEKO
-    #define fseeko fseek
-#endif
-
-#define AIO_KEEPOPEN 1
-#define AIO_NODELETE 2
-#define AIO_EOF 4
+#define AIO_BUF_LEN 256     /* read size for gets, read */
+#define AIO_WBUF_FULL_SIZE (64 * 1024)  /* This could be configurable */
+
+#define AIO_KEEPOPEN 1  /* don't set O_CLOEXEC, don't close on command delete */
+#define AIO_NODELETE 2  /* don't delete AF_UNIX path on close */
+#define AIO_EOF 4       /* EOF was reached */
+#define AIO_WBUF_NONE 8 /* default to buffering=none */
+#define AIO_NONBLOCK 16   /* socket is non-blocking */
+
+enum wbuftype {
+    WBUF_OPT_NONE,      /* write immediately */
+    WBUF_OPT_LINE,      /* write if NL is seen */
+    WBUF_OPT_FULL,      /* write when write buffer is full or on flush */
+};
 
 #if defined(JIM_IPV6)
 #define IPV6 1
@@ -118,13 +118,6 @@
 #define MAXPATHLEN JIM_PATH_LEN
 #endif
 
-#ifdef JIM_ANSIC
-/* no fdopen() with ANSIC, so can't support these */
-#undef HAVE_PIPE
-#undef HAVE_SOCKETPAIR
-#undef Jim_FileStat
-#endif
-
 #if defined(HAVE_SOCKETS) && !defined(JIM_BOOTSTRAP)
 /* Avoid type punned pointers */
 union sockaddr_any {
@@ -150,68 +143,99 @@ const char *inet_ntop(int af, const void *src, char *dst, int size)
 #endif
 #endif /* JIM_BOOTSTRAP */
 
+/* Wait for the fd to be readable and return JIM_OK if ok or JIM_ERR on timeout */
+/* ms=0 means block forever */
+static int JimReadableTimeout(int fd, long ms)
+{
+#ifdef HAVE_SELECT
+    int retval;
+    struct timeval tv;
+    fd_set rfds;
+
+    FD_ZERO(&rfds);
+
+    FD_SET(fd, &rfds);
+    tv.tv_sec = ms / 1000;
+    tv.tv_usec = (ms % 1000) * 1000;
+
+    retval = select(fd + 1, &rfds, NULL, NULL, ms == 0 ? NULL : &tv);
+
+    if (retval > 0) {
+        return JIM_OK;
+    }
+    return JIM_ERR;
+#else
+    return JIM_OK;
+#endif
+}
+
+
 struct AioFile;
 
 typedef struct {
     int (*writer)(struct AioFile *af, const char *buf, int len);
-    int (*reader)(struct AioFile *af, char *buf, int len);
-    const char *(*getline)(struct AioFile *af, char *buf, int len);
+    int (*reader)(struct AioFile *af, char *buf, int len, int pending);
     int (*error)(const struct AioFile *af);
     const char *(*strerror)(struct AioFile *af);
     int (*verify)(struct AioFile *af);
-    int (*eof)(struct AioFile *af);
-    int (*pending)(struct AioFile *af);
 } JimAioFopsType;
 
 typedef struct AioFile
 {
-    FILE *fp;
-    Jim_Obj *filename;
-    int type;
-    int flags;              /* AIO_KEEPOPEN? keep FILE* */
+    Jim_Obj *filename;      /* filename or equivalent for error reporting */
+    int wbuft;              /* enum wbuftype */
+    int flags;              /* AIO_KEEPOPEN | AIO_NODELETE | AIO_EOF */
+    long timeout;           /* timeout (in ms) for read operations if blocking */
     int fd;
     int addr_family;
     void *ssl;
     const JimAioFopsType *fops;
-    Jim_Obj *getline_partial;      /* In case of fgets() returning EAGAIN, partial line stored here */
+    Jim_Obj *readbuf;       /* Contains any buffered read data. NULL if empty. refcount=0 */
+    Jim_Obj *writebuf;      /* Contains any buffered write data. refcount=1 */
 } AioFile;
 
 static int stdio_writer(struct AioFile *af, const char *buf, int len)
 {
-    return fwrite(buf, 1, len, af->fp);
+    return write(af->fd, buf, len);
 }
 
-static int stdio_reader(struct AioFile *af, char *buf, int len)
+static int stdio_reader(struct AioFile *af, char *buf, int len, int nb)
 {
-    return fread(buf, 1, len, af->fp);
-}
+    if (nb || af->timeout == 0 || JimReadableTimeout(af->fd, af->timeout) == JIM_OK) {
+        /* timeout on blocking read */
+        int ret;
 
-static const char *stdio_getline(struct AioFile *af, char *buf, int len)
-{
-    return fgets(buf, len, af->fp);
+        errno = 0;
+        ret = read(af->fd, buf, len);
+        if (ret <= 0 && errno != EAGAIN && errno != EINTR) {
+            af->flags |= AIO_EOF;
+        }
+        return ret;
+    }
+    errno = ETIMEDOUT;
+    return -1;
 }
 
 static int stdio_error(const AioFile *af)
 {
-    if (!ferror(af->fp)) {
-        return JIM_OK;
-    }
-    clearerr(af->fp);
-    /* EAGAIN and similar are not error conditions. Just treat them like eof */
-    if (feof(af->fp) || errno == EAGAIN || errno == EINTR) {
+    if (af->flags & AIO_EOF) {
         return JIM_OK;
     }
+    /* XXX Probably errno should have been stashed in af->err instead */
+    switch (errno) {
+        case EAGAIN:
+        case EINTR:
+        case ETIMEDOUT:
 #ifdef ECONNRESET
-    if (errno == ECONNRESET) {
-        return JIM_OK;
-    }
+        case ECONNRESET:
 #endif
 #ifdef ECONNABORTED
-    if (errno == ECONNABORTED) {
-        return JIM_OK;
-    }
+        case ECONNABORTED:
 #endif
-    return JIM_ERR;
+            return JIM_OK;
+        default:
+            return JIM_ERR;
+    }
 }
 
 static const char *stdio_strerror(struct AioFile *af)
@@ -219,20 +243,12 @@ static const char *stdio_strerror(struct AioFile *af)
     return strerror(errno);
 }
 
-static int stdio_eof(struct AioFile *af)
-{
-    return feof(af->fp);
-}
-
 static const JimAioFopsType stdio_fops = {
     stdio_writer,
     stdio_reader,
-    stdio_getline,
     stdio_error,
     stdio_strerror,
     NULL, /* verify */
-    stdio_eof,
-    NULL, /* pending */
 };
 
 #if defined(JIM_SSL) && !defined(JIM_BOOTSTRAP)
@@ -244,65 +260,34 @@ static int ssl_writer(struct AioFile *af, const char *buf, int len)
     return SSL_write(af->ssl, buf, len);
 }
 
-static int ssl_pending(struct AioFile *af)
-{
-    return SSL_pending(af->ssl);
-}
-
-static int ssl_reader(struct AioFile *af, char *buf, int len)
+static int ssl_reader(struct AioFile *af, char *buf, int len, int nb)
 {
-    int ret = SSL_read(af->ssl, buf, len);
-    switch (SSL_get_error(af->ssl, ret)) {
-    case SSL_ERROR_NONE:
-        return ret;
-    case SSL_ERROR_SYSCALL:
-    case SSL_ERROR_ZERO_RETURN:
-        if (errno != EAGAIN) {
-            af->flags |= AIO_EOF;
+    if (nb || af->timeout == 0 || SSL_pending(af->ssl) ||  JimReadableTimeout(af->fd, af->timeout) == JIM_OK) {
+        int ret;
+        if (SSL_pending(af->ssl)) {
+            /* If there is pending data to read return it first */
+            if (len > SSL_pending(af->ssl)) {
+                len = SSL_pending(af->ssl);
+            }
         }
-        return 0;
-    case SSL_ERROR_SSL:
-    default:
-        if (errno == EAGAIN) {
-            return 0;
+        ret = SSL_read(af->ssl, buf, len);
+        if (ret <= 0 && errno != EAGAIN && errno != EINTR) {
+            af->flags |= AIO_EOF;
         }
-        af->flags |= AIO_EOF;
-        return -1;
+        return ret;
     }
-}
+    errno = ETIMEDOUT;
+    return -1;
 
-static int ssl_eof(struct AioFile *af)
-{
-    return (af->flags & AIO_EOF);
-}
-
-static const char *ssl_getline(struct AioFile *af, char *buf, int len)
-{
-    size_t i;
-    for (i = 0; i < len - 1 && !ssl_eof(af); i++) {
-        int ret = ssl_reader(af, &buf[i], 1);
-        if (ret != 1) {
-            break;
-        }
-        if (buf[i] == '\n') {
-            i++;
-            break;
-        }
-    }
-    buf[i] = '\0';
-    if (i == 0 && ssl_eof(af)) {
-        return NULL;
-    }
-    return buf;
 }
 
 static int ssl_error(const struct AioFile *af)
 {
     int ret = SSL_get_error(af->ssl, 0);
-	/* XXX should we be following the same logic as ssl_reader() here? */
-    if (ret == SSL_ERROR_ZERO_RETURN || ret == SSL_ERROR_NONE) {
-		return JIM_OK;
-	}
+    /* These indicate "normal" conditions */
+    if (ret == SSL_ERROR_ZERO_RETURN || ret == SSL_ERROR_NONE || ret == SSL_ERROR_WANT_READ) {
+            return JIM_OK;
+    }
     if (ret == SSL_ERROR_SYSCALL) {
         return stdio_error(af);
     }
@@ -341,18 +326,53 @@ static int ssl_verify(struct AioFile *af)
 static const JimAioFopsType ssl_fops = {
     ssl_writer,
     ssl_reader,
-    ssl_getline,
     ssl_error,
     ssl_strerror,
     ssl_verify,
-    ssl_eof,
-    ssl_pending,
 };
 #endif /* JIM_BOOTSTRAP */
 
+/**
+ * Sets nonblocking on the channel (if different from current)
+ * and updates the flags in af->flags.
+ */
+static void aio_set_nonblocking(AioFile *af, int nb)
+{
+#ifdef O_NDELAY
+    int old = !!(af->flags & AIO_NONBLOCK);
+    if (old != nb) {
+        int fmode = fcntl(af->fd, F_GETFL);
+        if (nb) {
+            fmode |= O_NDELAY;
+            af->flags |= AIO_NONBLOCK;
+        }
+        else {
+            fmode &= ~O_NDELAY;
+            af->flags &= ~AIO_NONBLOCK;
+        }
+        (void)fcntl(af->fd, F_SETFL, fmode);
+    }
+#endif
+}
+
+/**
+ * If the socket is blocking (not nonblocking) and a timeout is set,
+ * put the socket in non-blocking mode.
+ *
+ * Returns the original mode.
+ */
+static int aio_start_nonblocking(AioFile *af)
+{
+    int old = !!(af->flags & AIO_NONBLOCK);
+    if (af->timeout) {
+        aio_set_nonblocking(af, 1);
+    }
+    return old;
+}
+
 static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv);
-static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *filename,
-    const char *hdlfmt, int family, const char *mode, int flags);
+static AioFile *JimMakeChannel(Jim_Interp *interp, int fd, Jim_Obj *filename,
+    const char *hdlfmt, int family, int flags);
 
 #if defined(HAVE_SOCKETS) && !defined(JIM_BOOTSTRAP)
 #ifndef HAVE_GETADDRINFO
@@ -610,12 +630,23 @@ static int JimSetVariableSocketAddress(Jim_Interp *interp, Jim_Obj *varObjPtr, c
     return ret;
 }
 
-static Jim_Obj *aio_sockname(Jim_Interp *interp, AioFile *af)
+static Jim_Obj *aio_sockname(Jim_Interp *interp, int fd)
 {
     union sockaddr_any sa;
     socklen_t salen = sizeof(sa);
 
-    if (getsockname(af->fd, &sa.sa, &salen) < 0) {
+    if (getsockname(fd, &sa.sa, &salen) < 0) {
+        return NULL;
+    }
+    return JimFormatSocketAddress(interp, &sa, salen);
+}
+
+static Jim_Obj *aio_peername(Jim_Interp *interp, int fd)
+{
+    union sockaddr_any sa;
+    socklen_t salen = sizeof(sa);
+
+    if (getpeername(fd, &sa.sa, &salen) < 0) {
         return NULL;
     }
     return JimFormatSocketAddress(interp, &sa, salen);
@@ -642,10 +673,15 @@ static void JimAioSetError(Jim_Interp *interp, Jim_Obj *name)
     }
 }
 
+static int aio_eof(AioFile *af)
+{
+    return af->flags & AIO_EOF;
+}
+
 static int JimCheckStreamError(Jim_Interp *interp, AioFile *af)
 {
     int ret = 0;
-    if (!af->fops->eof(af)) {
+    if (!aio_eof(af)) {
         ret = af->fops->error(af);
         if (ret) {
             JimAioSetError(interp, af->filename);
@@ -654,16 +690,182 @@ static int JimCheckStreamError(Jim_Interp *interp, AioFile *af)
     return ret;
 }
 
+/**
+ * Removes n bytes from the beginning of objPtr.
+ *
+ * objPtr must have a string rep.
+ * n must be <= bytelen(objPtr)
+ */
+static void aio_consume(Jim_Obj *objPtr, int n)
+{
+    assert(objPtr->bytes);
+    assert(n <= objPtr->length);
+
+    /* Move the data down, plus 1 for the null terminator */
+    memmove(objPtr->bytes, objPtr->bytes + n, objPtr->length - n + 1);
+    objPtr->length -= n;
+    /* Note that we don't have to worry about utf8 len because the read and write
+     * buffers are used as pure byte buffers
+     */
+}
+
+/* forward declaration */
+static int aio_autoflush(Jim_Interp *interp, void *clientData, int mask);
+
+/**
+ * Flushes af->writebuf to the channel and removes that data
+ * from af->writebuf.
+ *
+ * If not all data could be written, starts a writable callback to continue
+ * flushing. This will only run when the eventloop does.
+ *
+ * On error or if not all data could be written, consumes only
+ * what was written and returns an error.
+ */
+static int aio_flush(Jim_Interp *interp, AioFile *af)
+{
+    int len;
+    const char *pt = Jim_GetString(af->writebuf, &len);
+    if (len) {
+        int ret = af->fops->writer(af, pt, len);
+        if (ret > 0) {
+            /* Consume what we wrote */
+            aio_consume(af->writebuf, ret);
+        }
+        if (ret < 0) {
+            return JimCheckStreamError(interp, af);
+        }
+        /* If not all data could be written, but with no error, and there is no writable
+         * handler, we can try to auto-flush
+         */
+        if (Jim_Length(af->writebuf)) {
+#ifdef jim_ext_eventloop
+            void *handler = Jim_FindFileHandler(interp, af->fd, JIM_EVENT_WRITABLE);
+            if (handler == NULL) {
+                Jim_CreateFileHandler(interp, af->fd, JIM_EVENT_WRITABLE, aio_autoflush, af, NULL);
+                return JIM_OK;
+            }
+            else if (handler == af) {
+                /* Nothing to do, handler already installed */
+                return JIM_OK;
+            }
+#endif
+            /* There is an existing foreign handler or no event loop so return an error */
+            Jim_SetResultString(interp, "send buffer is full", -1);
+            return JIM_ERR;
+        }
+    }
+    return JIM_OK;
+}
+
+/**
+ * Called when the channel is writable.
+ * Write what we can and return -1 when the write buffer is empty to remove the handler.
+ */
+static int aio_autoflush(Jim_Interp *interp, void *clientData, int mask)
+{
+    AioFile *af = clientData;
+
+    aio_flush(interp, af);
+    if (Jim_Length(af->writebuf) == 0) {
+        /* Done, so remove the handler */
+        return -1;
+    }
+    return 0;
+}
+
+/**
+ * Read until 'len' bytes are available in readbuf.
+ *
+ * If nonblocking or timeout, may return early.
+ * 'len' may be -1 to read until eof (or until no more data if nonblocking)
+ *
+ * Returns JIM_OK if data was read or JIM_ERR on error.
+ */
+static int aio_read_len(Jim_Interp *interp, AioFile *af, int nb, char *buf, size_t buflen, int neededLen)
+{
+    if (!af->readbuf) {
+        af->readbuf = Jim_NewStringObj(interp, NULL, 0);
+    }
+
+    if (neededLen >= 0) {
+        neededLen -= Jim_Length(af->readbuf);
+        if (neededLen <= 0) {
+            return JIM_OK;
+        }
+    }
+
+    while (neededLen && !aio_eof(af)) {
+        int retval;
+        int readlen;
+
+        if (neededLen == -1) {
+            readlen = AIO_BUF_LEN;
+        }
+        else {
+            readlen = (neededLen > AIO_BUF_LEN ? AIO_BUF_LEN : neededLen);
+        }
+        retval = af->fops->reader(af, buf, readlen, nb);
+        if (retval > 0) {
+            Jim_AppendString(interp, af->readbuf, buf, retval);
+            if (neededLen != -1) {
+                neededLen -= retval;
+            }
+            continue;
+        }
+        if (JimCheckStreamError(interp, af)) {
+            return JIM_ERR;
+        }
+        if (nb || af->timeout) {
+            return JIM_OK;
+        }
+    }
+
+    return JIM_OK;
+}
+
+/**
+ * Consumes neededLen bytes from readbuf and those
+ * bytes as a string object.
+ *
+ * If neededLen is -1, or >= len(readbuf), returns the entire readbuf.
+ *
+ * Returns NULL if no data available.
+ */
+static Jim_Obj *aio_read_consume(Jim_Interp *interp, AioFile *af, int neededLen)
+{
+    Jim_Obj *objPtr = NULL;
+
+    if (neededLen < 0 || af->readbuf == NULL || Jim_Length(af->readbuf) <= neededLen) {
+        objPtr = af->readbuf;
+        af->readbuf = NULL;
+    }
+    else if (af->readbuf) {
+        /* Need to consume part of the readbuf */
+        int len;
+        const char *pt = Jim_GetString(af->readbuf, &len);
+
+        objPtr  = Jim_NewStringObj(interp, pt, neededLen);
+        aio_consume(af->readbuf, neededLen);
+    }
+
+    return objPtr;
+}
+
 static void JimAioDelProc(Jim_Interp *interp, void *privData)
 {
     AioFile *af = privData;
 
     JIM_NOTUSED(interp);
 
+    /* Try to flush and write data before close */
+    aio_flush(interp, af);
+    Jim_DecrRefCount(interp, af->writebuf);
+
 #if UNIX_SOCKETS
     if (af->addr_family == PF_UNIX && (af->flags & AIO_NODELETE) == 0) {
         /* If this is bound, delete the socket file now */
-        Jim_Obj *filenameObj = aio_sockname(interp, af);
+        Jim_Obj *filenameObj = aio_sockname(interp, af->fd);
         if (filenameObj) {
             if (Jim_Length(filenameObj)) {
                 remove(Jim_String(filenameObj));
@@ -686,10 +888,10 @@ static void JimAioDelProc(Jim_Interp *interp, void *privData)
     }
 #endif
     if (!(af->flags & AIO_KEEPOPEN)) {
-        fclose(af->fp);
+        close(af->fd);
     }
-    if (af->getline_partial) {
-        Jim_FreeNewObj(interp, af->getline_partial);
+    if (af->readbuf) {
+        Jim_FreeNewObj(interp, af->readbuf);
     }
 
     Jim_Free(af);
@@ -698,14 +900,14 @@ static void JimAioDelProc(Jim_Interp *interp, void *privData)
 static int aio_cmd_read(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
-    char buf[AIO_BUF_LEN];
-    Jim_Obj *objPtr;
     int nonewline = 0;
-    int pending = 0;
     jim_wide neededLen = -1;         /* -1 is "read as much as possible" */
     static const char * const options[] = { "-pending", "-nonewline", NULL };
     enum { OPT_PENDING, OPT_NONEWLINE };
     int option;
+    int nb;
+    Jim_Obj *objPtr;
+    char buf[AIO_BUF_LEN];
 
     if (argc) {
         if (*Jim_String(argv[0]) == '-') {
@@ -714,11 +916,7 @@ static int aio_cmd_read(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
             }
             switch (option) {
                 case OPT_PENDING:
-                    if (!af->fops->pending) {
-                        Jim_SetResultString(interp, "-pending not supported on this connection type", -1);
-                        return JIM_ERR;
-                    }
-                    pending++;
+                    /* accepted for compatibility, but ignored */
                     break;
                 case OPT_NONEWLINE:
                     nonewline++;
@@ -739,86 +937,59 @@ static int aio_cmd_read(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     if (argc) {
         return -1;
     }
-    objPtr = Jim_NewStringObj(interp, NULL, 0);
-    while (neededLen != 0) {
-        int retval;
-        int readlen;
 
-        if (pending) {
-            readlen = 1;
-        }
-        else if (neededLen == -1) {
-            readlen = AIO_BUF_LEN;
-        }
-        else {
-            readlen = (neededLen > AIO_BUF_LEN ? AIO_BUF_LEN : neededLen);
-        }
-        retval = af->fops->reader(af, buf, readlen);
-        if (retval > 0) {
-            Jim_AppendString(interp, objPtr, buf, retval);
-            if (neededLen != -1) {
-                neededLen -= retval;
-            }
-            else if (pending) {
-                /* If pending was specified, after we do the initial read,
-                 * we do a second read to fetch any buffered data
-                 */
-                neededLen = af->fops->pending(af);
-                pending = 0;
-            }
-        }
-        if (retval <= 0) {
-            break;
-        }
-    }
-    /* Check for error conditions */
-    if (JimCheckStreamError(interp, af)) {
-        Jim_FreeNewObj(interp, objPtr);
+    /* reads are nonblocking if a timeout is given */
+    nb = aio_start_nonblocking(af);
+
+    if (aio_read_len(interp, af, nb, buf, sizeof(buf), neededLen) != JIM_OK) {
+        aio_set_nonblocking(af, nb);
         return JIM_ERR;
     }
-    if (nonewline) {
-        int len;
-        const char *s = Jim_GetString(objPtr, &len);
+    objPtr = aio_read_consume(interp, af, neededLen);
+
+    aio_set_nonblocking(af, nb);
+
+    if (objPtr) {
+        if (nonewline) {
+            int len;
+            const char *s = Jim_GetString(objPtr, &len);
 
-        if (len > 0 && s[len - 1] == '\n') {
-            objPtr->length--;
-            objPtr->bytes[objPtr->length] = '\0';
+            if (len > 0 && s[len - 1] == '\n') {
+                objPtr->length--;
+                objPtr->bytes[objPtr->length] = '\0';
+            }
         }
+        Jim_SetResult(interp, objPtr);
+    }
+    else {
+        Jim_SetEmptyResult(interp);
     }
-    Jim_SetResult(interp, objPtr);
     return JIM_OK;
 }
 
-AioFile *Jim_AioFile(Jim_Interp *interp, Jim_Obj *command)
+/* Use 'name getfd' to get the file descriptor associated with channel 'name'
+ * Currently this is only used by 'info channels'. Is there a better way?
+ */
+int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command)
 {
     Jim_Cmd *cmdPtr = Jim_GetCommand(interp, command, JIM_ERRMSG);
 
     /* XXX: There ought to be a supported API for this */
     if (cmdPtr && !cmdPtr->isproc && cmdPtr->u.native.cmdProc == JimAioSubCmdProc) {
-        return (AioFile *) cmdPtr->u.native.privData;
+        return ((AioFile *) cmdPtr->u.native.privData)->fd;
     }
     Jim_SetResultFormatted(interp, "Not a filehandle: \"%#s\"", command);
-    return NULL;
-}
-
-FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command)
-{
-    AioFile *af;
-
-    af = Jim_AioFile(interp, command);
-    if (af == NULL) {
-        return NULL;
-    }
-
-    return af->fp;
+    return -1;
 }
 
 static int aio_cmd_getfd(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
 
-    fflush(af->fp);
-    Jim_SetResultInt(interp, fileno(af->fp));
+    /* XXX Should we return this error? */
+    aio_flush(interp, af);
+
+    Jim_SetResultInt(interp, af->fd);
 
     return JIM_OK;
 }
@@ -828,16 +999,13 @@ static int aio_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     AioFile *af = Jim_CmdPrivData(interp);
     jim_wide count = 0;
     jim_wide maxlen = JIM_WIDE_MAX;
-    AioFile *outf = Jim_AioFile(interp, argv[0]);
     /* Small, static buffer for small files */
     char buf[AIO_BUF_LEN];
     /* Will be allocated if the file is large */
     char *bufp = buf;
     int buflen = sizeof(buf);
-
-    if (outf == NULL) {
-        return JIM_ERR;
-    }
+    int ok = 1;
+    Jim_Obj *objv[4];
 
     if (argc == 2) {
         if (Jim_GetWide(interp, argv[1], &maxlen) != JIM_OK) {
@@ -845,20 +1013,42 @@ static int aio_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         }
     }
 
+    /* Need to flush any write data first. This could fail because of send buf full,
+     * but more likely because the target isn't a filehandle.
+     * Should use use getfd to test for that case instead?
+     */
+    objv[0] = argv[0];
+    objv[1] = Jim_NewStringObj(interp, "flush", -1);
+    if (Jim_EvalObjVector(interp, 2, objv) != JIM_OK) {
+        Jim_SetResultFormatted(interp, "Not a filehandle: \"%#s\"", argv[0]);
+        return JIM_ERR;
+    }
+
+    /* Now prep for puts -nonewline. It's a shame we don't simply have 'write' */
+    objv[0] = argv[0];
+    objv[1] = Jim_NewStringObj(interp, "puts", -1);
+    objv[2] = Jim_NewStringObj(interp, "-nonewline", -1);
+    Jim_IncrRefCount(objv[1]);
+    Jim_IncrRefCount(objv[2]);
+
     while (count < maxlen) {
         jim_wide len = maxlen - count;
         if (len > buflen) {
             len = buflen;
         }
-
-        len = af->fops->reader(af, bufp, len);
-        if (len <= 0) {
+        if (aio_read_len(interp, af, 0, bufp, buflen, len) != JIM_OK) {
+            ok = 0;
+            break;
+        }
+        objv[3] = aio_read_consume(interp, af, len);
+        count += Jim_Length(objv[3]);
+        if (Jim_EvalObjVector(interp, 4, objv) != JIM_OK) {
+            ok = 0;
             break;
         }
-        if (outf->fops->writer(outf, bufp, len) != len) {
+        if (aio_eof(af)) {
             break;
         }
-        count += len;
         if (count >= 16384 && bufp == buf) {
             /* Heuristic check - for large copy speed-up */
             buflen = 65536;
@@ -870,7 +1060,10 @@ static int aio_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         Jim_Free(bufp);
     }
 
-    if (JimCheckStreamError(interp, af) || JimCheckStreamError(interp, outf)) {
+    Jim_DecrRefCount(interp, objv[1]);
+    Jim_DecrRefCount(interp, objv[2]);
+
+    if (!ok) {
         return JIM_ERR;
     }
 
@@ -883,54 +1076,54 @@ static int aio_cmd_gets(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
     char buf[AIO_BUF_LEN];
-    Jim_Obj *objPtr;
+    Jim_Obj *objPtr = NULL;
     int len;
-    int eof_or_partial = 0;
+    int nb;
+    char *nl = NULL;
+    int offset = 0;
 
     errno = 0;
 
-    if (af->getline_partial) {
-        /* A partial line was read previously, so append to it */
-        objPtr = af->getline_partial;
-        af->getline_partial = NULL;
-    }
-    else {
-        objPtr = Jim_NewStringObj(interp, NULL, 0);
-    }
+    /* reads are non-blocking if a timeout has been given */
+    nb = aio_start_nonblocking(af);
 
-    while (1) {
-        if (af->fops->getline(af, buf, AIO_BUF_LEN)) {
-            len = strlen(buf);
-            if (len && buf[len - 1] == '\n') {
-                /* strip "\n" and we are done */
-                Jim_AppendString(interp, objPtr, buf, len - 1);
-                break;
-            }
+    if (!af->readbuf) {
+        af->readbuf = Jim_NewStringObj(interp, NULL, 0);
+    }
 
-            /* Otherwise just append what we have */
-            Jim_AppendString(interp, objPtr, buf, len);
+    while (!aio_eof(af)) {
+        const char *pt = Jim_GetString(af->readbuf, &len);
+        nl = memchr(pt + offset, '\n', len - offset);
+        if (nl) {
+            /* got a line */
+            objPtr = Jim_NewStringObj(interp, pt, nl - pt);
+            /* And consume it plus the newline */
+            aio_consume(af->readbuf, nl - pt + 1);
+            break;
         }
 
-        if (errno == EAGAIN) {
-            if (Jim_Length(objPtr)) {
-                /* Stash the partial line */
-                af->getline_partial = objPtr;
-                /* And indicate that no line is (yet) available */
-                objPtr = Jim_NewStringObj(interp, NULL, 0);
+        offset = len;
+        len = af->fops->reader(af, buf, AIO_BUF_LEN, nb);
+        if (len <= 0) {
+            if (nb || af->timeout) {
+                /* Stop when no more to read (non-blocking) or timeout and return an empty string */
+                break;
             }
-            eof_or_partial = 1;
-            break;
         }
-        else if (af->fops->eof(af)) {
-            eof_or_partial = 1;
-            break;
+        else {
+            Jim_AppendString(interp, af->readbuf, buf, len);
         }
     }
 
-    if (Jim_Length(objPtr) == 0 && JimCheckStreamError(interp, af)) {
-        /* I/O error */
-        Jim_FreeNewObj(interp, objPtr);
-        return JIM_ERR;
+    aio_set_nonblocking(af, nb);
+
+    if (!nl && aio_eof(af)) {
+        /* Just take what we have as the line */
+        objPtr = af->readbuf;
+        af->readbuf = NULL;
+    }
+    else if (!objPtr) {
+        objPtr = Jim_NewStringObj(interp, NULL, 0);
     }
 
     if (argc) {
@@ -941,7 +1134,7 @@ static int aio_cmd_gets(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 
         len = Jim_Length(objPtr);
 
-        if (eof_or_partial && len == 0) {
+        if (!nl && len == 0) {
             /* On EOF or partial line with empty result, returns -1 if varName was specified */
             len = -1;
         }
@@ -959,32 +1152,61 @@ static int aio_cmd_puts(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     int wlen;
     const char *wdata;
     Jim_Obj *strObj;
+    int wnow = 0;
+    int nl = 1;
 
     if (argc == 2) {
         if (!Jim_CompareStringImmediate(interp, argv[0], "-nonewline")) {
             return -1;
         }
         strObj = argv[1];
+        nl = 0;
     }
     else {
         strObj = argv[0];
     }
 
-    wdata = Jim_GetString(strObj, &wlen);
-    if (af->fops->writer(af, wdata, wlen) == wlen) {
-        if (argc == 2 || af->fops->writer(af, "\n", 1) == 1) {
-            return JIM_OK;
-        }
+    /* Keep it simple and always go via the writebuf instead of trying to optimise
+     * the case that we can write immediately
+     */
+    Jim_AppendObj(interp, af->writebuf, strObj);
+    if (nl) {
+        Jim_AppendString(interp, af->writebuf, "\n", 1);
     }
-    JimAioSetError(interp, af->filename);
-    return JIM_ERR;
+
+    /* Now do we need to flush? */
+    wdata = Jim_GetString(af->writebuf, &wlen);
+    switch (af->wbuft) {
+        case WBUF_OPT_NONE:
+            /* Just write immediately */
+            wnow = 1;
+            break;
+
+        case WBUF_OPT_LINE:
+            /* Write everything if it contains a newline, or -nonewline wasn't given */
+            if (nl || memchr(wdata, '\n', wlen) != NULL) {
+                wnow = 1;
+            }
+            break;
+
+        case WBUF_OPT_FULL:
+            if (wlen >= AIO_WBUF_FULL_SIZE) {
+                wnow = 1;
+            }
+            break;
+    }
+
+    if (wnow) {
+        return aio_flush(interp, af);
+    }
+    return JIM_OK;
 }
 
 static int aio_cmd_isatty(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
 #ifdef HAVE_ISATTY
     AioFile *af = Jim_CmdPrivData(interp);
-    Jim_SetResultInt(interp, isatty(fileno(af->fp)));
+    Jim_SetResultInt(interp, isatty(af->fd));
 #else
     Jim_SetResultInt(interp, 0);
 #endif
@@ -1008,7 +1230,7 @@ static int aio_cmd_recvfrom(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 
     buf = Jim_Alloc(len + 1);
 
-    rlen = recvfrom(fileno(af->fp), buf, len, 0, &sa.sa, &salen);
+    rlen = recvfrom(af->fd, buf, len, 0, &sa.sa, &salen);
     if (rlen < 0) {
         Jim_Free(buf);
         JimAioSetError(interp, NULL);
@@ -1041,7 +1263,7 @@ static int aio_cmd_sendto(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     wdata = Jim_GetString(argv[0], &wlen);
 
     /* Note that we don't validate the socket type. Rely on sendto() failing if appropriate */
-    len = sendto(fileno(af->fp), wdata, wlen, 0, &sa.sa, salen);
+    len = sendto(af->fd, wdata, wlen, 0, &sa.sa, salen);
     if (len < 0) {
         JimAioSetError(interp, NULL);
         return JIM_ERR;
@@ -1050,12 +1272,18 @@ static int aio_cmd_sendto(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     return JIM_OK;
 }
 
+/**
+ * Returns the peer name of 'fd' or NULL on error.
+ */
+
+
 static int aio_cmd_accept(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
     int sock;
     union sockaddr_any sa;
     socklen_t salen = sizeof(sa);
+    Jim_Obj *filenameObj;
 
     sock = accept(af->fd, &sa.sa, &salen);
     if (sock < 0) {
@@ -1069,15 +1297,21 @@ static int aio_cmd_accept(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         }
     }
 
+    /* This probably can't fail at this point */
+    filenameObj = JimFormatSocketAddress(interp, &sa, salen);
+    if (!filenameObj) {
+        filenameObj = Jim_NewStringObj(interp, "accept", -1);
+    }
+
     /* Create the file command */
-    return JimMakeChannel(interp, NULL, sock, Jim_NewStringObj(interp, "accept", -1),
-        "aio.sockstream%ld", af->addr_family, "r+", AIO_NODELETE) ? JIM_OK : JIM_ERR;
+    return JimMakeChannel(interp, sock, filenameObj,
+        "aio.sockstream%ld", af->addr_family, AIO_NODELETE) ? JIM_OK : JIM_ERR;
 }
 
 static int aio_cmd_sockname(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
-    Jim_Obj *objPtr = aio_sockname(interp, af);
+    Jim_Obj *objPtr = aio_sockname(interp, af->fd);
 
     if (objPtr == NULL) {
         JimAioSetError(interp, NULL);
@@ -1090,14 +1324,13 @@ static int aio_cmd_sockname(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 static int aio_cmd_peername(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
-    union sockaddr_any sa;
-    socklen_t salen = sizeof(sa);
+    Jim_Obj *objPtr = aio_peername(interp, af->fd);
 
-    if (getpeername(af->fd, &sa.sa, &salen) < 0) {
+    if (objPtr == NULL) {
         JimAioSetError(interp, NULL);
         return JIM_ERR;
     }
-    Jim_SetResult(interp, JimFormatSocketAddress(interp, &sa, salen));
+    Jim_SetResult(interp, objPtr);
     return JIM_OK;
 }
 
@@ -1122,19 +1355,14 @@ static int aio_cmd_listen(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 static int aio_cmd_flush(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
-
-    if (fflush(af->fp) == EOF) {
-        JimAioSetError(interp, af->filename);
-        return JIM_ERR;
-    }
-    return JIM_OK;
+    return aio_flush(interp, af);
 }
 
 static int aio_cmd_eof(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
 
-    Jim_SetResultInt(interp, !!af->fops->eof(af));
+    Jim_SetResultInt(interp, !!aio_eof(af));
     return JIM_OK;
 }
 
@@ -1198,10 +1426,19 @@ static int aio_cmd_seek(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     if (Jim_GetWide(interp, argv[0], &offset) != JIM_OK) {
         return JIM_ERR;
     }
-    if (fseeko(af->fp, offset, orig) == -1) {
+    if (orig != SEEK_CUR || offset != 0) {
+        /* Try to write flush if seeking. XXX What about on error? */
+        aio_flush(interp, af);
+    }
+    if (Jim_Lseek(af->fd, offset, orig) == -1) {
         JimAioSetError(interp, af->filename);
         return JIM_ERR;
     }
+    if (af->readbuf) {
+        Jim_FreeNewObj(interp, af->readbuf);
+        af->readbuf = NULL;
+    }
+    af->flags &= ~AIO_EOF;
     return JIM_OK;
 }
 
@@ -1209,7 +1446,7 @@ static int aio_cmd_tell(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
 
-    Jim_SetResultInt(interp, ftello(af->fp));
+    Jim_SetResultInt(interp, Jim_Lseek(af->fd, 0, SEEK_CUR));
     return JIM_OK;
 }
 
@@ -1222,19 +1459,6 @@ static int aio_cmd_filename(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 }
 
 #ifdef O_NDELAY
-static int aio_set_nonblocking(int fd, int nb)
-{
-    int fmode = fcntl(fd, F_GETFL);
-    if (nb) {
-        fmode |= O_NDELAY;
-    }
-    else {
-        fmode &= ~O_NDELAY;
-    }
-    (void)fcntl(fd, F_SETFL, fmode);
-    return fmode;
-}
-
 static int aio_cmd_ndelay(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
@@ -1245,10 +1469,9 @@ static int aio_cmd_ndelay(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         if (Jim_GetLong(interp, argv[0], &nb) != JIM_OK) {
             return JIM_ERR;
         }
-        aio_set_nonblocking(af->fd, nb);
+        aio_set_nonblocking(af, nb);
     }
-    int fmode = fcntl(af->fd, F_GETFL);
-    Jim_SetResultInt(interp, (fmode & O_NONBLOCK) ? 1 : 0);
+    Jim_SetResultInt(interp, (af->flags & AIO_NONBLOCK) ? 1 : 0);
     return JIM_OK;
 }
 #endif
@@ -1359,7 +1582,6 @@ static int aio_cmd_sync(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     AioFile *af = Jim_CmdPrivData(interp);
 
-    fflush(af->fp);
     fsync(af->fd);
     return JIM_OK;
 }
@@ -1375,31 +1597,35 @@ static int aio_cmd_buffering(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         "full",
         NULL
     };
-    enum
-    {
-        OPT_NONE,
-        OPT_LINE,
-        OPT_FULL,
-    };
-    int option;
 
-    if (Jim_GetEnum(interp, argv[0], options, &option, NULL, JIM_ERRMSG) != JIM_OK) {
+    if (Jim_GetEnum(interp, argv[0], options, &af->wbuft, NULL, JIM_ERRMSG) != JIM_OK) {
         return JIM_ERR;
     }
-    switch (option) {
-        case OPT_NONE:
-            setvbuf(af->fp, NULL, _IONBF, 0);
-            break;
-        case OPT_LINE:
-            setvbuf(af->fp, NULL, _IOLBF, BUFSIZ);
-            break;
-        case OPT_FULL:
-            setvbuf(af->fp, NULL, _IOFBF, BUFSIZ);
-            break;
+
+    if (af->wbuft == WBUF_OPT_NONE) {
+        return aio_flush(interp, af);
     }
+    /* don't bother flushing when switching from full to line */
     return JIM_OK;
 }
 
+static int aio_cmd_timeout(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
+{
+#ifdef HAVE_SELECT
+    AioFile *af = Jim_CmdPrivData(interp);
+    if (argc == 1) {
+        if (Jim_GetLong(interp, argv[0], &af->timeout) != JIM_OK) {
+            return JIM_ERR;
+        }
+    }
+    Jim_SetResultInt(interp, af->timeout);
+    return JIM_OK;
+#else
+    Jim_SetResultString(interp, "timeout not supported", -1);
+    return JIM_ERR;
+#endif
+}
+
 #ifdef jim_ext_eventloop
 static int aio_eventinfo(Jim_Interp *interp, AioFile * af, unsigned mask,
     int argc, Jim_Obj * const *argv)
@@ -1511,7 +1737,7 @@ static int aio_cmd_ssl(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 
     SSL_set_cipher_list(ssl, "ALL");
 
-    if (SSL_set_fd(ssl, fileno(af->fp)) == 0) {
+    if (SSL_set_fd(ssl, af->fd) == 0) {
         goto out;
     }
 
@@ -1677,7 +1903,7 @@ static int aio_cmd_tty(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 
 static const jim_subcmd_type aio_command_table[] = {
     {   "read",
-        "?-nonewline|-pending|len?",
+        "?-nonewline|len?",
         aio_cmd_read,
         0,
         2,
@@ -1868,6 +2094,13 @@ static const jim_subcmd_type aio_command_table[] = {
         1,
         /* Description: Returns script, or invoke exception-script when oob data, {} to remove */
     },
+    {   "timeout",
+        "?ms?",
+        aio_cmd_timeout,
+        0,
+        1,
+        /* Description: Timeout for blocking read, gets */
+    },
 #endif
 #if !defined(JIM_BOOTSTRAP)
 #if defined(JIM_SSL)
@@ -1921,11 +2154,107 @@ static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     return Jim_CallSubCmd(interp, Jim_ParseSubCmd(interp, aio_command_table, argc, argv), argc, argv);
 }
 
+/**
+ * Returns open flags or 0 on error.
+ */
+static int parse_posix_open_mode(Jim_Interp *interp, Jim_Obj *modeObj)
+{
+    int i;
+    int flags = 0;
+    #ifndef O_NOCTTY
+        /* mingw doesn't support this flag */
+        #define O_NOCTTY 0
+    #endif
+    static const char * const modetypes[] = {
+        "RDONLY", "WRONLY", "RDWR", "APPEND", "BINARY", "CREAT", "EXCL", "NOCTTY", "TRUNC", NULL
+    };
+    static const int modeflags[] = {
+        O_RDONLY, O_WRONLY, O_RDWR, O_APPEND, 0, O_CREAT, O_EXCL, O_NOCTTY, O_TRUNC,
+    };
+
+    for (i = 0; i < Jim_ListLength(interp, modeObj); i++) {
+        int opt;
+        Jim_Obj *objPtr = Jim_ListGetIndex(interp, modeObj, i);
+        if (Jim_GetEnum(interp, objPtr, modetypes, &opt, "access mode", JIM_ERRMSG) != JIM_OK) {
+            return -1;
+        }
+        flags |= modeflags[opt];
+    }
+    return flags;
+}
+
+/**
+ * Returns flags for open() or -1 on error and sets an error.
+ */
+static int parse_open_mode(Jim_Interp *interp, Jim_Obj *filenameObj, Jim_Obj *modeObj)
+{
+    /* Parse the specified mode. */
+    int flags;
+    const char *mode = Jim_String(modeObj);
+    if (*mode == 'R' || *mode == 'W') {
+        return parse_posix_open_mode(interp, modeObj);
+    }
+    if (*mode == 'r') {
+        flags = O_RDONLY;
+    }
+    else if (*mode == 'w') {
+        flags = O_WRONLY | O_CREAT | O_TRUNC;
+    }
+    else if (*mode == 'a') {
+        flags = O_WRONLY | O_CREAT | O_APPEND;
+    }
+    else {
+        Jim_SetResultFormatted(interp, "%s: invalid open mode '%s'", Jim_String(filenameObj), mode);
+        return -1;
+    }
+    mode++;
+
+    if (*mode == 'b') {
+#ifdef O_BINARY
+        flags |= O_BINARY;
+#endif
+        mode++;
+    }
+
+    if (*mode == 't') {
+#ifdef O_TEXT
+        flags |= O_TEXT;
+#endif
+        mode++;
+    }
+
+    if (*mode == '+') {
+        mode++;
+        /* read+write so set O_RDWR instead */
+        flags &= ~(O_RDONLY | O_WRONLY);
+        flags |= O_RDWR;
+    }
+
+    if (*mode == 'x') {
+        mode++;
+#ifdef O_EXCL
+        flags |= O_EXCL;
+#endif
+    }
+
+    if (*mode == 'F') {
+        mode++;
+#ifdef O_LARGEFILE
+        flags |= O_LARGEFILE;
+#endif
+    }
+
+    if (*mode == 'e') {
+        /* ignore close on exec since this is the default */
+        mode++;
+    }
+    return flags;
+}
+
 static int JimAioOpenCommand(Jim_Interp *interp, int argc,
         Jim_Obj *const *argv)
 {
-    const char *mode;
-    FILE *fh = NULL;
+    int flags;
     const char *filename;
     int fd = -1;
 
@@ -1935,7 +2264,6 @@ static int JimAioOpenCommand(Jim_Interp *interp, int argc,
     }
 
     filename = Jim_String(argv[1]);
-    mode = (argc == 3) ? Jim_String(argv[2]) : "r";
 
 #ifdef jim_ext_tclcompat
     {
@@ -1943,70 +2271,34 @@ static int JimAioOpenCommand(Jim_Interp *interp, int argc,
         /* If the filename starts with '|', use popen instead */
         if (*filename == '|') {
             Jim_Obj *evalObj[3];
+            int n = 0;
 
-            evalObj[0] = Jim_NewStringObj(interp, "::popen", -1);
-            evalObj[1] = Jim_NewStringObj(interp, filename + 1, -1);
-            evalObj[2] = Jim_NewStringObj(interp, mode, -1);
+            evalObj[n++] = Jim_NewStringObj(interp, "::popen", -1);
+            evalObj[n++] = Jim_NewStringObj(interp, filename + 1, -1);
+            if (argc == 3) {
+                evalObj[n++] = argv[2];
+            }
 
-            return Jim_EvalObjVector(interp, 3, evalObj);
+            return Jim_EvalObjVector(interp, n, evalObj);
         }
     }
 #endif
-#ifndef JIM_ANSIC
-    if (*mode == 'R' || *mode == 'W') {
-        /* POSIX flags */
-        #ifndef O_NOCTTY
-            /* mingw doesn't support this flag */
-            #define O_NOCTTY 0
-        #endif
-        static const char * const modetypes[] = {
-            "RDONLY", "WRONLY", "RDWR", "APPEND", "BINARY", "CREAT", "EXCL", "NOCTTY", "TRUNC", NULL
-        };
-        static const char * const simplemodes[] = {
-            "r", "w", "w+"
-        };
-        static const int modeflags[] = {
-            O_RDONLY, O_WRONLY, O_RDWR, O_APPEND, 0, O_CREAT, O_EXCL, O_NOCTTY, O_TRUNC,
-        };
-        int posixflags = 0;
-        int len = Jim_ListLength(interp, argv[2]);
-        int i;
-        int opt;
-
-        mode = NULL;
-
-        for (i = 0; i < len; i++) {
-            Jim_Obj *objPtr = Jim_ListGetIndex(interp, argv[2], i);
-            if (Jim_GetEnum(interp, objPtr, modetypes, &opt, "access mode", JIM_ERRMSG) != JIM_OK) {
-                return JIM_ERR;
-            }
-            if (opt < 3) {
-                mode = simplemodes[opt];
-            }
-            posixflags |= modeflags[opt];
-        }
-        /* mode must be set here if it started with 'R' or 'W' and passed the enum check above */
-        assert(mode);
-        fd = open(filename, posixflags, 0666);
-        if (fd >= 0) {
-            fh = fdopen(fd, mode);
-            if (fh == NULL) {
-                close(fd);
-            }
+    if (argc == 3) {
+        flags = parse_open_mode(interp, argv[1], argv[2]);
+        if (flags == -1) {
+            return JIM_ERR;
         }
     }
-    else
-#endif
-    {
-        fh = fopen(filename, mode);
+    else {
+        flags = O_RDONLY;
     }
-
-    if (fh == NULL) {
+    fd = open(filename, flags, 0666);
+    if (fd < 0) {
         JimAioSetError(interp, argv[1]);
         return JIM_ERR;
     }
 
-    return JimMakeChannel(interp, fh, fd, argv[1], "aio.handle%ld", 0, mode, 0) ? JIM_OK : JIM_ERR;
+    return JimMakeChannel(interp, fd, argv[1], "aio.handle%ld", 0, 0) ? JIM_OK : JIM_ERR;
 }
 
 #if defined(JIM_SSL) && !defined(JIM_BOOTSTRAP)
@@ -2039,10 +2331,9 @@ static SSL_CTX *JimAioSslCtx(Jim_Interp *interp)
 #endif /* JIM_BOOTSTRAP */
 
 /**
- * Creates a channel for fh/fd/filename.
+ * Creates a channel for fd/filename.
  *
- * If fh is not NULL, uses that as the channel (and sets AIO_KEEPOPEN).
- * Otherwise fd must be >= 0, in which case it uses that as the channel.
+ * fd must be a valid file descriptor.
  *
  * hdlfmt is a sprintf format for the filehandle. Anything with %ld at the end will do.
  * mode is used for open or fdopen.
@@ -2050,26 +2341,13 @@ static SSL_CTX *JimAioSslCtx(Jim_Interp *interp)
  * Creates the command and sets the name as the current result.
  * Returns the AioFile pointer on sucess or NULL on failure (only if fdopen fails).
  */
-static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *filename,
-    const char *hdlfmt, int family, const char *mode, int flags)
+static AioFile *JimMakeChannel(Jim_Interp *interp, int fd, Jim_Obj *filename,
+    const char *hdlfmt, int family, int flags)
 {
     AioFile *af;
     char buf[AIO_CMD_LEN];
     Jim_Obj *cmdname;
 
-    if (fh == NULL) {
-        assert(fd >= 0);
-#ifndef JIM_ANSIC
-        fh = fdopen(fd, mode);
-
-        if (fh == NULL) {
-            JimAioSetError(interp, filename);
-            close(fd);
-            return NULL;
-        }
-#endif
-    }
-
     snprintf(buf, sizeof(buf), hdlfmt, Jim_GetId(interp));
     cmdname = Jim_NewStringObj(interp, buf, -1);
     if (!filename) {
@@ -2080,20 +2358,33 @@ static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *fi
     /* Create the file command */
     af = Jim_Alloc(sizeof(*af));
     memset(af, 0, sizeof(*af));
-    af->fp = fh;
     af->filename = filename;
-    af->flags = flags;
-#ifndef JIM_ANSIC
-    af->fd = fileno(fh);
+    af->fd = fd;
+    af->addr_family = family;
+    af->fops = &stdio_fops;
+    af->ssl = NULL;
+    if (flags & AIO_WBUF_NONE) {
+        af->wbuft = WBUF_OPT_NONE;
+    }
+    else {
+#ifdef HAVE_ISATTY
+        af->wbuft = isatty(af->fd) ? WBUF_OPT_LINE : WBUF_OPT_FULL;
+#else
+        af->wbuft = WBUF_OPT_FULL;
+#endif
+    }
+    /* don't set flags yet so that aio_set_nonblocking() works */
 #ifdef FD_CLOEXEC
     if ((flags & AIO_KEEPOPEN) == 0) {
         (void)fcntl(af->fd, F_SETFD, FD_CLOEXEC);
     }
 #endif
-#endif
-    af->addr_family = family;
-    af->fops = &stdio_fops;
-    af->ssl = NULL;
+    aio_set_nonblocking(af, !!(flags & AIO_NONBLOCK));
+    /* Now set flags */
+    af->flags |= flags;
+    /* Create an empty write buf */
+    af->writebuf = Jim_NewStringObj(interp, NULL, 0);
+    Jim_IncrRefCount(af->writebuf);
 
     Jim_CreateCommand(interp, buf, JimAioSubCmdProc, af, JimAioDelProc);
 
@@ -2110,12 +2401,12 @@ static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *fi
  * Create a pair of channels. e.g. from pipe() or socketpair()
  */
 static int JimMakeChannelPair(Jim_Interp *interp, int p[2], Jim_Obj *filename,
-    const char *hdlfmt, int family, const char * const mode[2])
+    const char *hdlfmt, int family, int flags)
 {
-    if (JimMakeChannel(interp, NULL, p[0], filename, hdlfmt, family, mode[0], 0)) {
+    if (JimMakeChannel(interp, p[0], filename, hdlfmt, family, flags)) {
         Jim_Obj *objPtr = Jim_NewListObj(interp, NULL, 0);
         Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp));
-        if (JimMakeChannel(interp, NULL, p[1], filename, hdlfmt, family, mode[1], 0)) {
+        if (JimMakeChannel(interp, p[1], filename, hdlfmt, family, flags)) {
             Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp));
             Jim_SetResult(interp, objPtr);
             return JIM_OK;
@@ -2134,7 +2425,6 @@ static int JimMakeChannelPair(Jim_Interp *interp, int p[2], Jim_Obj *filename,
 static int JimAioPipeCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 {
     int p[2];
-    static const char * const mode[2] = { "r", "w" };
 
     if (argc != 1) {
         Jim_WrongNumArgs(interp, 1, argv, "");
@@ -2146,7 +2436,7 @@ static int JimAioPipeCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         return JIM_ERR;
     }
 
-    return JimMakeChannelPair(interp, p, argv[0], "aio.pipe%ld", 0, mode);
+    return JimMakeChannelPair(interp, p, argv[0], "aio.pipe%ld", 0, 0);
 }
 #endif
 
@@ -2155,7 +2445,6 @@ static int JimAioOpenPtyCommand(Jim_Interp *interp, int argc, Jim_Obj *const *ar
 {
     int p[2];
     char path[MAXPATHLEN];
-    static const char * const mode[2] = { "r+", "w+" };
 
     if (argc != 1) {
         Jim_WrongNumArgs(interp, 1, argv, "");
@@ -2168,7 +2457,7 @@ static int JimAioOpenPtyCommand(Jim_Interp *interp, int argc, Jim_Obj *const *ar
     }
 
     /* Note: The replica path will be used for both handles slave */
-    return JimMakeChannelPair(interp, p, Jim_NewStringObj(interp, path, -1), "aio.pty%ld", 0, mode);
+    return JimMakeChannelPair(interp, p, Jim_NewStringObj(interp, path, -1), "aio.pty%ld", 0, 0);
 }
 #endif
 
@@ -2220,6 +2509,7 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
     Jim_Obj *argv0 = argv[0];
     int ipv6 = 0;
     int async = 0;
+    int flags = 0;
 
 
     while (argc > 1 && Jim_String(argv[1])[0] == '-') {
@@ -2232,7 +2522,7 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         }
         switch (option) {
             case OPT_ASYNC:
-                async = 1;
+                flags |= AIO_NONBLOCK;
                 break;
 
             case OPT_IPV6:
@@ -2261,13 +2551,12 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
 
     if (argc > 2) {
         addr = Jim_String(argv[2]);
-		filename = argv[2];
+        filename = argv[2];
     }
 
 #if defined(HAVE_SOCKETPAIR) && UNIX_SOCKETS
     if (socktype == SOCK_STREAM_SOCKETPAIR) {
         int p[2];
-        static const char * const mode[2] = { "r+", "r+" };
 
         if (addr || ipv6) {
             goto wrongargs;
@@ -2277,7 +2566,8 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
             JimAioSetError(interp, NULL);
             return JIM_ERR;
         }
-        return JimMakeChannelPair(interp, p, argv[1], "aio.sockpair%ld", PF_UNIX, mode);
+        /* Should we expect socketpairs to be line buffered by default? */
+        return JimMakeChannelPair(interp, p, argv[1], "aio.sockpair%ld", PF_UNIX, 0);
     }
 #endif
 
@@ -2387,9 +2677,6 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
         JimAioSetError(interp, NULL);
         return JIM_ERR;
     }
-    if (async) {
-        aio_set_nonblocking(sock, 1);
-    }
     if (bind_addr) {
         if (JimParseSocketAddress(interp, family, type, bind_addr, &sa, &salen) != JIM_OK) {
             close(sock);
@@ -2427,11 +2714,11 @@ static int JimAioSockCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
             return JIM_ERR;
         }
     }
-	if (!filename) {
-		filename = argv[1];
-	}
+    if (!filename) {
+        filename = argv[1];
+    }
 
-    return JimMakeChannel(interp, NULL, sock, filename, "aio.sock%ld", family, "r+", 0) ? JIM_OK : JIM_ERR;
+    return JimMakeChannel(interp, sock, filename, "aio.sock%ld", family, flags) ? JIM_OK : JIM_ERR;
 }
 #endif /* JIM_BOOTSTRAP */
 
@@ -2475,9 +2762,9 @@ int Jim_aioInit(Jim_Interp *interp)
 #endif
 
     /* Create filehandles for stdin, stdout and stderr */
-    JimMakeChannel(interp, stdin, -1, NULL, "stdin", 0, "r", AIO_KEEPOPEN);
-    JimMakeChannel(interp, stdout, -1, NULL, "stdout", 0, "w", AIO_KEEPOPEN);
-    JimMakeChannel(interp, stderr, -1, NULL, "stderr", 0, "w", AIO_KEEPOPEN);
+    JimMakeChannel(interp, fileno(stdin), NULL, "stdin", 0, AIO_KEEPOPEN);
+    JimMakeChannel(interp, fileno(stdout), NULL, "stdout", 0, AIO_KEEPOPEN);
+    JimMakeChannel(interp, fileno(stderr), NULL, "stderr", 0, AIO_KEEPOPEN | AIO_WBUF_NONE);
 
     return JIM_OK;
 }
diff --git a/jim-eventloop.h b/jim-eventloop.h
index f2b2abf..ee0b28d 100644
--- a/jim-eventloop.h
+++ b/jim-eventloop.h
@@ -74,6 +74,8 @@ JIM_EXPORT jim_wide Jim_CreateTimeHandler (Jim_Interp *interp,
         Jim_EventFinalizerProc *finalizerProc);
 JIM_EXPORT jim_wide Jim_DeleteTimeHandler (Jim_Interp *interp, jim_wide id);
 JIM_EXPORT void *Jim_FindFileHandler(Jim_Interp *interp, int fd, int mask);
+/* This should probably be in jimiocompat.h */
+JIM_EXPORT int Jim_ReadableTimeout(int fd, long ms);
 
 #define JIM_FILE_EVENTS 1
 #define JIM_TIME_EVENTS 2
diff --git a/jim-exec.c b/jim-exec.c
index 69a7841..343d5e1 100644
--- a/jim-exec.c
+++ b/jim-exec.c
@@ -129,24 +129,19 @@ static void Jim_RemoveTrailingNewline(Jim_Obj *objPtr)
 static int JimAppendStreamToString(Jim_Interp *interp, int fd, Jim_Obj *strObj)
 {
     char buf[256];
-    FILE *fh = fdopen(fd, "r");
     int ret = 0;
 
-    if (fh == NULL) {
-        return -1;
-    }
-
     while (1) {
-        int retval = fread(buf, 1, sizeof(buf), fh);
+        int retval = read(fd, buf, sizeof(buf));
         if (retval > 0) {
             ret = 1;
             Jim_AppendString(interp, strObj, buf, retval);
         }
-        if (retval != sizeof(buf)) {
+        if (retval <= 0) {
             break;
         }
     }
-    fclose(fh);
+    close(fd);
     return ret;
 }
 
@@ -441,7 +436,7 @@ static int Jim_ExecCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv)
      */
     if (errorId != -1) {
         int ret;
-        lseek(errorId, 0, SEEK_SET);
+        Jim_Lseek(errorId, 0, SEEK_SET);
         ret = JimAppendStreamToString(interp, errorId, errStrObj);
         if (ret < 0) {
             Jim_SetResultErrno(interp, "error reading from error pipe");
@@ -872,7 +867,7 @@ badargs:
                 close(inputId);
                 goto error;
             }
-            lseek(inputId, 0L, SEEK_SET);
+            Jim_Lseek(inputId, 0L, SEEK_SET);
         }
         else if (inputFile == FILE_HANDLE) {
             int fd = JimGetChannelFd(interp, input);
diff --git a/jim-format.c b/jim-format.c
index f5f0f92..d036117 100644
--- a/jim-format.c
+++ b/jim-format.c
@@ -41,6 +41,7 @@
  */
 #include <ctype.h>
 #include <string.h>
+#include <stdio.h>
 
 #include <jim.h>
 #include "utf8.h"
diff --git a/jim-interactive.c b/jim-interactive.c
index c6490ec..abdf491 100644
--- a/jim-interactive.c
+++ b/jim-interactive.c
@@ -1,5 +1,6 @@
 #include <errno.h>
 #include <string.h>
+#include <stdio.h>
 
 #include "jimautoconf.h"
 #include <jim.h>
diff --git a/jim-interp.c b/jim-interp.c
index cf02d5f..90e2474 100644
--- a/jim-interp.c
+++ b/jim-interp.c
@@ -1,4 +1,5 @@
 #include <assert.h>
+#include <stdio.h>
 
 #include "jim.h"
 #include "jimautoconf.h"
diff --git a/jim-load.c b/jim-load.c
index b6be26d..480d677 100644
--- a/jim-load.c
+++ b/jim-load.c
@@ -1,4 +1,5 @@
 #include <string.h>
+#include <stdio.h>
 
 #include "jimautoconf.h"
 #include <jim.h>
diff --git a/jim-package.c b/jim-package.c
index be53688..69af074 100644
--- a/jim-package.c
+++ b/jim-package.c
@@ -1,4 +1,5 @@
 #include <string.h>
+#include <stdio.h>
 
 #include "jimautoconf.h"
 #include <jim-subcmd.h>
diff --git a/jim.c b/jim.c
index 908cc31..0e21f29 100644
--- a/jim.c
+++ b/jim.c
@@ -11539,39 +11539,52 @@ int Jim_EvalFileGlobal(Jim_Interp *interp, const char *filename)
 }
 
 #include <sys/stat.h>
+#include "jimiocompat.h"
 
-int Jim_EvalFile(Jim_Interp *interp, const char *filename)
+/**
+ * Reads the text file contents into an object and returns with a zero ref count.
+ * Returns NULL and sets an error if can't read the file.
+ */
+static Jim_Obj *JimReadTextFile(Jim_Interp *interp, const char *filename)
 {
-    FILE *fp;
+    jim_stat_t sb;
+    int fd;
     char *buf;
-    Jim_Obj *scriptObjPtr;
-    struct stat sb;
-    int retcode;
     int readlen;
 
-    if (stat(filename, &sb) != 0 || (fp = fopen(filename, "rt")) == NULL) {
+    if (Jim_Stat(filename, &sb) == -1 || (fd = open(filename, O_RDONLY | O_TEXT, 0666)) < 0) {
         Jim_SetResultFormatted(interp, "couldn't read file \"%s\": %s", filename, strerror(errno));
-        return JIM_ERR;
-    }
-    if (sb.st_size == 0) {
-        fclose(fp);
-        return JIM_OK;
+        return NULL;
     }
-
     buf = Jim_Alloc(sb.st_size + 1);
-    readlen = fread(buf, 1, sb.st_size, fp);
-    if (ferror(fp)) {
-        fclose(fp);
+    readlen = read(fd, buf, sb.st_size);
+    close(fd);
+    if (readlen < 0) {
         Jim_Free(buf);
         Jim_SetResultFormatted(interp, "failed to load file \"%s\": %s", filename, strerror(errno));
-        return JIM_ERR;
+        return NULL;
+    }
+    else {
+        Jim_Obj *objPtr;
+        buf[readlen] = 0;
+
+        objPtr = Jim_NewStringObjNoAlloc(interp, buf, readlen);
+
+        return objPtr;
     }
-    fclose(fp);
-    buf[readlen] = 0;
+}
+
+
+int Jim_EvalFile(Jim_Interp *interp, const char *filename)
+{
+    Jim_Obj *scriptObjPtr;
+    int retcode;
 
-    scriptObjPtr = Jim_NewStringObjNoAlloc(interp, buf, readlen);
+    scriptObjPtr = JimReadTextFile(interp, filename);
+    if (!scriptObjPtr) {
+        return JIM_ERR;
+    }
     JimSetSourceInfo(interp, scriptObjPtr, Jim_NewStringObj(interp, filename, -1), 1);
-    Jim_IncrRefCount(scriptObjPtr);
 
     retcode = Jim_EvalObj(interp, scriptObjPtr);
 
@@ -11584,8 +11597,6 @@ int Jim_EvalFile(Jim_Interp *interp, const char *filename)
         }
     }
 
-    Jim_DecrRefCount(interp, scriptObjPtr);
-
     return retcode;
 }
 
@@ -11806,7 +11817,7 @@ static void JimCommandMatch(Jim_Interp *interp, Jim_Obj *listObjPtr,
 
     Jim_IncrRefCount(keyObj);
 
-    if (type != JIM_CMDLIST_CHANNELS || Jim_AioFilehandle(interp, keyObj)) {
+    if (type != JIM_CMDLIST_CHANNELS || Jim_AioFilehandle(interp, keyObj) >= 0) {
         int match = 1;
         if (patternObj) {
             int plen, slen;
@@ -16677,10 +16688,9 @@ int Jim_PackageProvide(Jim_Interp *interp, const char *name, const char *ver, in
 }
 #endif
 #ifndef jim_ext_aio
-FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *fhObj)
+int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *fhObj)
 {
-    Jim_SetResultString(interp, "aio not enabled", -1);
-    return NULL;
+    return -1;
 }
 #endif
 
diff --git a/jim.h b/jim.h
index 0d5ab0a..1c4b5c8 100644
--- a/jim.h
+++ b/jim.h
@@ -71,7 +71,6 @@ extern "C" {
 
 #include <time.h>
 #include <limits.h>
-#include <stdio.h>  /* for the FILE typedef definition */
 #include <stdlib.h> /* In order to export the Jim_Free() macro */
 #include <stdarg.h> /* In order to get type va_list */
 
@@ -975,7 +974,7 @@ JIM_EXPORT int Jim_LoadLibrary(Jim_Interp *interp, const char *pathName);
 JIM_EXPORT void Jim_FreeLoadHandles(Jim_Interp *interp);
 
 /* jim-aio.c */
-JIM_EXPORT FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command);
+JIM_EXPORT int Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command);
 
 /* type inspection - avoid where possible */
 JIM_EXPORT int Jim_IsDict(Jim_Obj *objPtr);
diff --git a/jimiocompat.h b/jimiocompat.h
index 64aa4ff..0837b73 100644
--- a/jimiocompat.h
+++ b/jimiocompat.h
@@ -68,6 +68,7 @@ int Jim_OpenForRead(const char *filename);
     typedef struct __stat64 jim_stat_t;
     #define Jim_Stat _stat64
     #define Jim_FileStat _fstat64
+    #define Jim_Lseek _lseeki64
 
 #else
     #if defined(HAVE_STAT64)
@@ -89,6 +90,11 @@ int Jim_OpenForRead(const char *filename);
             #define Jim_LinkStat lstat
         #endif
     #endif
+    #if defined(HAVE_LSEEK64)
+        #define Jim_Lseek lseek64
+    #else
+        #define Jim_Lseek lseek
+    #endif
 
     #if defined(HAVE_UNISTD_H)
         #include <unistd.h>
@@ -107,6 +113,10 @@ int Jim_OpenForRead(const char *filename);
     #endif
 #endif
 
+#ifndef O_TEXT
+#define O_TEXT 0
+#endif
+
 /* jim-file.c */
 /* Note that this is currently an internal function only.
  * It does not form part of the public Jim API
diff --git a/make-bootstrap-jim b/make-bootstrap-jim
index cf8138d..315d610 100755
--- a/make-bootstrap-jim
+++ b/make-bootstrap-jim
@@ -101,6 +101,7 @@ cat <<EOF
 #define HAVE_DIRENT_H
 #define HAVE_UNISTD_H
 #define HAVE_UMASK
+#define HAVE_PIPE
 #endif
 EOF
 
diff --git a/tests/aio.test b/tests/aio.test
index a012518..f6852b9 100644
--- a/tests/aio.test
+++ b/tests/aio.test
@@ -8,11 +8,20 @@ testConstraint posixaio [expr {$tcl_platform(platform) eq {unix} && !$tcl_platfo
 set f [open testdata.in wb]
 $f puts test-data
 $f close
+# create a test file file with several lines
+set f [open copy.in wb]
+$f puts line1
+$f puts line2
+$f puts line3
+$f close
+
 set f [open testdata.in rb]
 
 defer {
 	$f close
 	file delete testdata.in
+	file delete copy.in
+	file delete copy.out
 }
 
 test aio-1.1 {seek usage} -body {
@@ -72,11 +81,7 @@ test aio-2.3 {read -ve len} -body {
 
 test aio-2.4 {read too many args} -body {
 	$f read 20 extra
-} -returnCodes error -match glob -result {wrong # args: should be "* read ?-nonewline|-pending|len?"}
-
-test aio-2.5 {read -pending on non-ssl} -body {
-	$f read -pending
-} -returnCodes error -result {-pending not supported on this connection type}
+} -returnCodes error -match glob -result {wrong # args: should be "* read ?-nonewline|len?"}
 
 test aio-3.1 {copy to invalid fh} -body {
 	$f copy lambda
@@ -137,7 +142,7 @@ test aio-9.1 {open: posix modes} -constraints posixaio -body {
 
 test aio-9.2 {open: posix modes, bad modes} -constraints posixaio -body {
 	open testdata.in {CREAT TRUNC}
-} -returnCodes error -result {testdata.in: Invalid argument}
+} -returnCodes error -result {testdata.in: invalid open mode 'CREAT TRUNC'}
 
 test aio-9.3 {open: posix modes, bad modes} -constraints posixaio -body {
 	open testdata.in {WRONG TRUNC}
@@ -156,4 +161,166 @@ test aio-9.4 {open: posix modes} -constraints posixaio -cleanup {
 	set buf
 } -result {write-data}
 
+test copyto-1.1 {basic copyto} {
+	set in [open copy.in]
+	set out [open copy.out w]
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {line1 line2 line3}
+
+test copyto-1.2 {copyto with limit} {
+	set in [open copy.in]
+	set out [open copy.out w]
+	$in copyto $out 8
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {line1 li {}}
+
+test copyto-1.3 {copyto after gets} {
+	set in [open copy.in]
+	set out [open copy.out w]
+	$in gets
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {line2 line3 {}}
+
+test copyto-1.4 {copyto after read} {
+	set in [open copy.in]
+	$in read 3
+	set out [open copy.out w]
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {e1 line2 line3}
+
+test copyto-1.5 {copyto after gets, seek} {
+	set in [open copy.in]
+	$in gets
+	$in seek 2 start
+	set out [open copy.out w]
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {ne1 line2 line3}
+
+test copyto-1.6 {copyto from pipe} {
+	set in [open "|cat copy.in"]
+	set out [open copy.out w]
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {line1 line2 line3}
+
+test copyto-1.6 {copyto to pipe} {
+	set out [open "|cat >copy.out" w]
+	set in [open copy.in]
+	$in copyto $out
+	$in close
+	$out close
+	set ff [open copy.out]
+	set result [list [$ff gets] [$ff gets] [$ff gets]]
+	$ff close
+	set result
+} {line1 line2 line3}
+
+# Creates a child process and returns {pid writehandle}
+# The child expects to read $numlines lines of input and exits with a return
+# code of 0 if ok
+proc child_reader {numlines} {
+	# create a pipe with the child as a slightly slow reader
+	lassign [socket pipe] r w
+
+	set pid [os.fork]
+	if {$pid == 0} {
+		# child
+		$w close
+		# sleep a moment to make sure the parent fills up the send buffer
+		sleep 0.5
+		set n 0
+		while {[$r gets buf] >= 0} {
+			incr n
+		}
+		#puts "child got $n/$numlines lines"
+		$r close
+		if {$n == $numlines} {
+			# This is what we expect
+			exit 99
+		}
+		# This is not expected
+		exit 98
+	}
+	# parent
+	$r close
+
+	list $pid $w
+}
+
+test autoflush-1.1 {pipe writer, blocking} -constraints socket -body {
+	lassign [child_reader 10000] pid w
+
+	# Send data fast enough to fill up the send buffer
+	loop i 10000 {
+		$w puts "this is line $i"
+	}
+
+	# No autoflush needed. The write won't return
+	# until queued
+	$w close
+
+	lassign [wait $pid] - - rc
+
+	list $rc
+} -result {99}
+
+test autoflush-1.2 {pipe writer, non blocking} -constraints socket -body {
+	lassign [child_reader 10000] pid w
+
+	$w ndelay 1
+
+	# Send data fast enough to fill up the send buffer
+	# With older jimtcl this would return an error "pipe: Resource temporarily unavailable"
+	loop i 10000 {
+		$w puts "this is line $i"
+	}
+
+	# Now data should still be queued, wait for autoflush
+	lassign [time {
+		after idle {}
+		vwait done
+	}] t1
+
+	# puts "autoflush finished in ${t1}us, closing pipe"
+	$w close
+
+	lassign [wait $pid] - - rc
+
+	list $rc $t1
+} -match glob -result {99 *}
+
 testreport
diff --git a/tests/runall.tcl b/tests/runall.tcl
index 96a56a9..5c9aa8b 100644
--- a/tests/runall.tcl
+++ b/tests/runall.tcl
@@ -47,11 +47,11 @@ if {[info commands interp] eq ""} {
 				puts [format "%16s:   --- error ($msg)" $script]
 				incr total(fail)
 			} elseif {[info return $opts(-code)] eq "exit"} {
-				# if the test explicitly called exit 99,
+				# if the test explicitly called exit 98 or 99,
 				# it must be from a child process via os.fork, so
-				# silently exit
-				if {$msg eq "99"} {
-					exit 0
+				# silently exit with that return code
+				if {$msg in {98 99}} {
+					exit $msg
 				}
 			}
 
diff --git a/tests/socket.test b/tests/socket.test
index 67fdb9c..1eb98b4 100644
--- a/tests/socket.test
+++ b/tests/socket.test
@@ -129,6 +129,8 @@ test socket-1.6 {pipe} -body {
 
 test socket-1.7 {socketpair} -body {
 	lassign [socket pair] s1 s2
+	$s1 buffering line
+	$s2 buffering line
 	stdout flush
 	if {[os.fork] == 0} {
 		$s1 close
@@ -338,20 +340,18 @@ set s [socket stream.server 0]
 if {[os.fork] == 0} {
 	# child
 	set c [socket stream [socket-connect-addr $s]]
-	# Note: We have to disable buffering here, otherwise
-	# when we read data in $c readable {} we many leave buffered
-	# data and readable won't retrigger.
-	$c buffering none
 	$s close
+	$c ndelay 1
 	$c readable {
-		# when we read we need to also read any pending data,
-		# otherwise readable won't retrigger
-		set buf [$c read 1]
-		if {[string length $buf] == 0} {
+		# read everything available (non-blocking read)
+		set buf [$c read]
+		if {[string length $buf]} {
+			$c puts -nonewline $buf
+			$c flush
+		}
+		if {[$c eof]} {
 			incr readdone
 			$c close
-		} else {
-			$c puts -nonewline $buf
 		}
 	}
 	vwait readdone
@@ -365,6 +365,8 @@ defer {
 }
 $s close
 
+$cs buffering line
+
 # At this point, $cs is the server connection to the client in the child process
 
 test eventloop-1.1 {puts/gets} {
@@ -372,14 +374,67 @@ test eventloop-1.1 {puts/gets} {
 	$cs gets
 } hello
 
-test eventloop-1.2 {puts/gets} {
+test eventloop-1.2 {puts/read} {
 	$cs puts -nonewline again
+	$cs flush
 	lmap p [range 5] {
 		set c [$cs read 1]
 		set c
 	}
 } {a g a i n}
 
+test eventloop-1.3 {gets with no timeout and multiple newlines} {
+	$cs puts a\nb\nc\nd\ne
+	lmap p [range 5] {
+		$cs gets buf
+		set buf
+	}
+} {a b c d e}
+
+test eventloop-1.4 {gets with timeout and multiple newlines} {
+	$cs timeout 100
+	$cs puts a\nb\nc\nd\ne
+	lmap p [range 6] {
+		set rc [$cs gets buf]
+		set buf
+	}
+} {a b c d e {}}
+
+test eventloop-1.5 {gets with timeout and incomplete line} {
+	$cs timeout 100
+	$cs puts -nonewline first
+	list [$cs gets buf] $buf
+} {-1 {}}
+
+test eventloop-1.6 {gets with timeout and complete line} {
+	$cs timeout 100
+	$cs puts second
+	list [$cs gets buf] $buf
+} {11 firstsecond}
+
+test eventloop-1.7 {gets when read with extra data} {
+	$cs timeout 100
+	$cs puts -nonewline abcde
+	$cs flush
+	# This won't get get a line 
+	$cs gets line
+	# now read should read the data
+	set data [$cs read -nonewline]
+	list $line $data
+} {{} abcde}
+
+test eventloop-1.7 {read with timeout and no data} {
+	$cs timeout 100
+	$cs read
+} {}
+
+test eventloop-1.6 {read with timeout and data} {
+	$cs timeout 100
+	$cs puts -nonewline data
+	$cs flush
+	$cs read
+} {data}
+
 test sockopt-1.1 {sockopt} -body {
 	lsort [dict keys [$cs sockopt]]
 } -match glob -result {*tcp_nodelay*}
diff --git a/tests/ssl.test b/tests/ssl.test
index b01069d..d147c92 100644
--- a/tests/ssl.test
+++ b/tests/ssl.test
@@ -17,16 +17,16 @@ if {[os.fork] == 0} {
 	# child
 	set c [[socket stream 127.0.0.1:1443] ssl]
 	$s close
+	$c ndelay 1
 	sleep 0.25
 	$c readable {
-		# when we read we need to also read any pending data,
-		# otherwise readable won't retrigger
-		set buf [$c read -pending]
-		if {[string length $buf] == 0} {
+		# read everything available and echo it back
+		set buf [$c read]
+		$c puts -nonewline $buf
+		$c flush
+		if {[$c eof]} {
 			incr ssldone
 			$c close
-		} else {
-			$c puts -nonewline $buf
 		}
 	}
 	vwait ssldone
@@ -42,6 +42,7 @@ defer {
 }
 
 # At this point, $cs is the server connection to the client in the child process
+$cs buffering line
 
 test ssl-1.1 {puts/gets} {
 	$cs puts hello
@@ -50,6 +51,7 @@ test ssl-1.1 {puts/gets} {
 
 test ssl-1.2 {puts/gets} {
 	$cs puts -nonewline again
+	$cs flush
 	lmap p [range 5] {
 		set c [$cs read 1]
 		set c