Re: [Qemu-devel] [RFC v3 PATCH 07/45] multi-process: define proxy-link object

[Stefan Hajnoczi]

On Tue, Sep 03, 2019 at 04:37:33PM -0400, Jagannathan Raman wrote:
> diff --git a/include/io/proxy-link.h b/include/io/proxy-link.h
> new file mode 100644
> index 0000000..ee78cdd
> --- /dev/null
> +++ b/include/io/proxy-link.h
> @@ -0,0 +1,147 @@

Regarding naming: so far I've seen "remote", "mpqemu", and "proxy".
These concepts aren't well-defined and I'm not sure if they refer to the
same thing or are different.  ProxyLinkState should be named
MPQemuLinkState?

It's also unclear to me how tightly coupled this "proxy link" is to PCI
(since it has PCI Configuration Space commands and how it would be
cleanly extended to support other busses).

> +/*
> + * Communication channel between QEMU and remote device process
> + *
> + * Copyright 2019, Oracle and/or its affiliates.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> + * of this software and associated documentation files (the "Software"), to 
> deal
> + * in the Software without restriction, including without limitation the 
> rights
> + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> + * copies of the Software, and to permit persons to whom the Software is
> + * furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> + * THE SOFTWARE.
> + */
> +
> +#ifndef PROXY_LINK_H
> +#define PROXY_LINK_H
> +
> +#include <stddef.h>
> +#include <stdint.h>
> +#include <glib.h>
> +#include <pthread.h>
> +
> +#include "qemu/osdep.h"
> +#include "qom/object.h"
> +#include "qemu/thread.h"
> +
> +typedef struct ProxyLinkState ProxyLinkState;
> +
> +#define TYPE_PROXY_LINK "proxy-link"
> +#define PROXY_LINK(obj) \
> +    OBJECT_CHECK(ProxyLinkState, (obj), TYPE_PROXY_LINK)
> +
> +#define REMOTE_MAX_FDS 8
> +
> +#define PROC_HDR_SIZE offsetof(ProcMsg, data1.u64)
> +
> +/*
> + * proc_cmd_t enum type to specify the command to be executed on the remote
> + * device
> + *
> + * Following commands are supported:
> + * CONF_READ        PCI config. space read
> + * CONF_WRITE       PCI config. space write
> + *
> + */
> +typedef enum {
> +    INIT = 0,
> +    CONF_READ,
> +    CONF_WRITE,
> +    MAX,
> +} proc_cmd_t;
> +
> +/*
> + * ProcMsg Format of the message sent to the remote device from QEMU
> + *
> + * cmd         The remote command
> + * bytestream  Indicates if the data to be shared is structured (data1)
> + *             or unstructured (data2)
> + * size        Size of the data to be shared
> + * data1       Structured data
> + * fds         File descriptors to be shared with remote device
> + * data2       Unstructured data
> + *
> + */
> +typedef struct {
> +    proc_cmd_t cmd;
> +    int bytestream;

Please use bool.

> +    size_t size;
> +
> +    union {
> +        uint64_t u64;
> +    } data1;
> +
> +    int fds[REMOTE_MAX_FDS];
> +    int num_fds;
> +
> +    uint8_t *data2;
> +} ProcMsg;
> +
> +struct conf_data_msg {
> +    uint32_t addr;
> +    uint32_t val;
> +    int l;

Please pick a descriptive name for this field.

> +};
> +
> +/*
> + * ProcChannel defines the channel that make up the communication link
> + * between QEMU and remote process
> + *
> + * gsrc       GSource object to be used by loop
> + * gpfd       GPollFD object containing the socket & events to monitor
> + * sock       Socket to send/receive communication, same as the one in gpfd
> + * lock       Mutex to synchronize access to the channel
> + */

Please see the doc comment style in include/qom/object.h for examples of
how to format doc comments.

> +
> +typedef struct ProcChannel {
> +    GSource gsrc;
> +    GPollFD gpfd;
> +    int sock;
> +    QemuMutex lock;
> +} ProcChannel;
> +
> +typedef void (*proxy_link_callback)(GIOCondition cond, ProcChannel *chan);
> +
> +/*
> + * ProxyLinkState Instance info. of the communication
> + * link between QEMU and remote process. The Link could
> + * be made up of multiple channels.
> + *
> + * ctx        GMainContext to be used for communication
> + * loop       Main loop that would be used to poll for incoming data
> + * com        Communication channel to transport control messages
> + *
> + */
> +struct ProxyLinkState {
> +    Object obj;
> +
> +    GMainContext *ctx;
> +    GMainLoop *loop;
> +
> +    ProcChannel *com;
> +
> +    proxy_link_callback callback;
> +};
> +
> +ProxyLinkState *proxy_link_create(void);
> +void proxy_link_finalize(ProxyLinkState *s);
> +
> +void proxy_proc_send(ProxyLinkState *s, ProcMsg *msg, ProcChannel *chan);
> +int proxy_proc_recv(ProxyLinkState *s, ProcMsg *msg, ProcChannel *chan);
> +
> +void proxy_link_init_channel(ProxyLinkState *s, ProcChannel **chan, int fd);
> +void proxy_link_destroy_channel(ProcChannel *chan);
> +void proxy_link_set_callback(ProxyLinkState *s, proxy_link_callback 
> callback);
> +void start_handler(ProxyLinkState *s);
> +
> +#endif
> diff --git a/io/Makefile.objs b/io/Makefile.objs
> index 9a20fce..ff88b46 100644
> --- a/io/Makefile.objs
> +++ b/io/Makefile.objs
> @@ -10,3 +10,5 @@ io-obj-y += channel-util.o
>  io-obj-y += dns-resolver.o
>  io-obj-y += net-listener.o
>  io-obj-y += task.o
> +
> +io-obj-$(CONFIG_MPQEMU) += proxy-link.o
> diff --git a/io/proxy-link.c b/io/proxy-link.c
> new file mode 100644
> index 0000000..5eb9718
> --- /dev/null
> +++ b/io/proxy-link.c
> @@ -0,0 +1,292 @@
> +/*
> + * Communication channel between QEMU and remote device process
> + *
> + * Copyright 2019, Oracle and/or its affiliates.
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> + * of this software and associated documentation files (the "Software"), to 
> deal
> + * in the Software without restriction, including without limitation the 
> rights
> + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> + * copies of the Software, and to permit persons to whom the Software is
> + * furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice shall be included in
> + * all copies or substantial portions of the Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
> + * THE SOFTWARE.
> + */
> +
> +#include <assert.h>
> +#include <errno.h>
> +#include <pthread.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <sys/types.h>
> +#include <sys/socket.h>
> +#include <sys/un.h>
> +#include <unistd.h>
> +
> +#include "qemu/module.h"
> +#include "io/proxy-link.h"
> +#include "qemu/log.h"
> +
> +GSourceFuncs gsrc_funcs;
> +
> +static void proxy_link_inst_init(Object *obj)
> +{
> +    ProxyLinkState *s = PROXY_LINK(obj);
> +
> +    s->ctx = g_main_context_default();
> +    s->loop = g_main_loop_new(s->ctx, FALSE);
> +}
> +
> +static const TypeInfo proxy_link_info = {
> +    .name = TYPE_PROXY_LINK,
> +    .parent = TYPE_OBJECT,
> +    .instance_size = sizeof(ProxyLinkState),
> +    .instance_init = proxy_link_inst_init,
> +};
> +
> +static void proxy_link_register_types(void)
> +{
> +    type_register_static(&proxy_link_info);
> +}
> +
> +type_init(proxy_link_register_types)
> +
> +ProxyLinkState *proxy_link_create(void)
> +{
> +    return PROXY_LINK(object_new(TYPE_PROXY_LINK));
> +}
> +
> +void proxy_link_finalize(ProxyLinkState *s)
> +{
> +    g_main_loop_unref(s->loop);
> +    g_main_context_unref(s->ctx);
> +    g_main_loop_quit(s->loop);
> +
> +    proxy_link_destroy_channel(s->com);
> +
> +    object_unref(OBJECT(s));
> +}

It's strange that the destruction of ProxyLinkState does not use QEMU's
object system.  How about implementing .instance_finalize(), dropping
proxy_link_finalize(), and letting the caller unref ProxyLinkState like
a regular Object?

> +
> +void proxy_proc_send(ProxyLinkState *s, ProcMsg *msg, ProcChannel *chan)
> +{
> +    int rc;
> +    uint8_t *data;
> +    union {
> +        char control[CMSG_SPACE(REMOTE_MAX_FDS * sizeof(int))];
> +        struct cmsghdr align;
> +    } u;
> +    struct msghdr hdr;
> +    struct cmsghdr *chdr;
> +    int sock = chan->sock;
> +    QemuMutex *lock = &chan->lock;
> +
> +    struct iovec iov = {
> +        .iov_base = (char *) msg,
> +        .iov_len = PROC_HDR_SIZE,
> +    };
> +
> +    memset(&hdr, 0, sizeof(hdr));
> +    memset(&u, 0, sizeof(u));
> +
> +    hdr.msg_iov = &iov;
> +    hdr.msg_iovlen = 1;
> +
> +    if (msg->num_fds > REMOTE_MAX_FDS) {
> +        qemu_log_mask(LOG_REMOTE_DEBUG, "%s: Max FDs exceeded\n", __func__);
> +        return;
> +    }
> +
> +    if (msg->num_fds > 0) {
> +        size_t fdsize = msg->num_fds * sizeof(int);
> +
> +        hdr.msg_control = &u;
> +        hdr.msg_controllen = sizeof(u);
> +
> +        chdr = CMSG_FIRSTHDR(&hdr);
> +        chdr->cmsg_len = CMSG_LEN(fdsize);
> +        chdr->cmsg_level = SOL_SOCKET;
> +        chdr->cmsg_type = SCM_RIGHTS;
> +        memcpy(CMSG_DATA(chdr), msg->fds, fdsize);
> +        hdr.msg_controllen = CMSG_SPACE(fdsize);
> +    }
> +
> +    qemu_mutex_lock(lock);
> +
> +    do {
> +        rc = sendmsg(sock, &hdr, 0);
> +    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
> +
> +    if (rc < 0) {
> +        qemu_log_mask(LOG_REMOTE_DEBUG, "%s - sendmsg rc is %d, errno is %d,"
> +                      " sock %d\n", __func__, rc, errno, sock);
> +        qemu_mutex_unlock(lock);
> +        return;
> +    }
> +
> +    if (msg->bytestream) {
> +        data = msg->data2;
> +    } else {
> +        data = (uint8_t *)msg + PROC_HDR_SIZE;
> +    }
> +
> +    do {
> +        rc = write(sock, data, msg->size);
> +    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
> +
> +    qemu_mutex_unlock(lock);

Can this message be transmitted in a single sendmsg() by including 2
iovecs instead of just 1?  Then no locking is needed and only one
syscall is required.

> +}
> +
> +
> +int proxy_proc_recv(ProxyLinkState *s, ProcMsg *msg, ProcChannel *chan)
> +{
> +    int rc;
> +    uint8_t *data;
> +    union {
> +        char control[CMSG_SPACE(REMOTE_MAX_FDS * sizeof(int))];
> +        struct cmsghdr align;
> +    } u;
> +    struct msghdr hdr;
> +    struct cmsghdr *chdr;
> +    size_t fdsize;
> +    int sock = chan->sock;
> +    QemuMutex *lock = &chan->lock;
> +
> +    struct iovec iov = {
> +        .iov_base = (char *) msg,
> +        .iov_len = PROC_HDR_SIZE,
> +    };
> +
> +    memset(&hdr, 0, sizeof(hdr));
> +    memset(&u, 0, sizeof(u));
> +
> +    hdr.msg_iov = &iov;
> +    hdr.msg_iovlen = 1;
> +    hdr.msg_control = &u;
> +    hdr.msg_controllen = sizeof(u);
> +
> +    qemu_mutex_lock(lock);

Sockets are full-duplex (sending and receiving are independent).  Is it
necessary to use the same lock for both send and receive?

> +
> +    do {
> +        rc = recvmsg(sock, &hdr, 0);
> +    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
> +
> +    if (rc < 0) {
> +        qemu_log_mask(LOG_REMOTE_DEBUG, "%s - recvmsg rc is %d, errno is %d,"
> +                      " sock %d\n", __func__, rc, errno, sock);
> +        qemu_mutex_unlock(lock);
> +        return rc;
> +    }
> +
> +    msg->num_fds = 0;
> +    for (chdr = CMSG_FIRSTHDR(&hdr); chdr != NULL;
> +         chdr = CMSG_NXTHDR(&hdr, chdr)) {
> +        if ((chdr->cmsg_level == SOL_SOCKET) &&
> +            (chdr->cmsg_type == SCM_RIGHTS)) {
> +            fdsize = chdr->cmsg_len - CMSG_LEN(0);
> +            msg->num_fds = fdsize / sizeof(int);
> +            memcpy(msg->fds, CMSG_DATA(chdr), fdsize);

Please validate num_fds before memcpy to prevent the buffer overflow.

> +            break;
> +        }
> +    }
> +
> +    if (msg->size && msg->bytestream) {
> +        msg->data2 = calloc(1, msg->size);
> +        data = msg->data2;
> +    } else {
> +        data = (uint8_t *)&msg->data1;
> +    }
> +
> +    if (msg->size) {
> +        do {
> +            rc = read(sock, data, msg->size);
> +        } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
> +    }

Please validate size to prevent the buffer overflow.

signature.asc
Description: PGP signature