PKG_CHECK_MODULES([DISTLIBS], [libconfuse >= 2.5 libcurl libxml-2.0 liblist >= 2.3.1 libarchive >= 2.8.0 libcrypto])

 * Submits a file to distren.

 *

 * Blocks.

 *

 * @param handle distren handle.

 * @param job must not refer to a valid job.

 * @param filename path to a tarball representing a job.

 * @return 0 on success, 1 on error.

 */

#include <errno.h>

#include <libgen.h>

#ifdef _WIN32

#define FOPEN_BINARY "b"

#else

#define FOPEN_BINARY ""

#endif

  FILE *in;

  char buf[DISTREN_REQUEST_FILE_POST_DATA_LEN];

  size_t len;

  void *data;

  struct distren_request *req;

  struct distren_request_file_post data_header;

  distren_request_file_post_context_t post_context;

  /* for basename() to play with */

  char *my_filename;

  errno = 0;

  in = fopen(filename, "r" FOPEN_BINARY);

  if(!in)

    {

      perror("fopen");

      return 1;

    }

  fprintf(stderr, "info: Starting upload of %s...\n", filename);

  /* prepare the server for a file upload */

  my_filename = strdup(filename);

  distren_request_file_post_start(&req,

				  &data,

				  &post_context,

				  handle->file_post_id,

				  basename(my_filename));

  free(my_filename);

  distren_request_send(handle->rem, req, data);

  distren_request_free_with_data(req, data);

  handle->file_post_id ++;

  /* send file body */

  while(in

	&& !ferror(in)

	&& !feof(in)

	&& handle->rem)

    {

      len = fread(buf, 1, DISTREN_REQUEST_FILE_POST_DATA_LEN, in);

      if(len == 0)

	continue;

      distren_request_file_post(&req,

				&data_header,

				post_context,

				buf,

				len);

      remoteio_write(handle->rem, req, sizeof(struct distren_request));

      /* should we check of handle->rem is NULL or not here...? */

      remoteio_write(handle->rem, &data_header, sizeof(struct distren_request_file_post));

      remoteio_write(handle->rem, buf, len);

      distren_request_free(req);

      /* ensure we have no more than a megabyte waiting to be sent. */

      while(handle->rem

	    && remoteio_sendq_len(handle->rem) / 3 > (1024 * 1024 / DISTREN_REQUEST_FILE_POST_DATA_LEN))

	{

	  fprintf(stderr, "info: %d packets waiting to be sent...\n", remoteio_sendq_len(handle->rem) / 3);

	  multiio_poll(handle->multiio, -1);

	}

    }

  /*

   * declare the upload as finished, marking it as cancelled on I/O

   * error.

  */

  distren_request_file_post_finish(&req, &data, post_context, (!in || ferror(in)) ? 1 : 0);

  if(handle->rem)

    distren_request_send(handle->rem, req, data);

    distren_request_free_with_data(req, data);

  if(in)

    fclose(in);

  if(!handle->rem)

    return 1;

  /* let's block until the file's gone. */

  while(handle->rem

	&& remoteio_sendq_len(handle->rem))

    {

      fprintf(stderr, "info: %d packets waiting to be sent...\n", remoteio_sendq_len(handle->rem) / 3);

      multiio_poll(handle->multiio, -1);

    }

  fprintf(stderr, "info: %s successfully uploaded, as far as we know.\n", filename);

  return 0;

  /*

   * The last-used per-connection DISTREN_REQUEST_FILE_POST

   * identifier.

   */

  uint32_t file_post_id;

#include <openssl/evp.h>

     * Declare that a client is preparing to post a file using a

     * particular post-id and with a user-friendly filename.

     *

     * Yes, this is evil. Yes, this tries to replicate

     * well-established protocols like FTP and HTTP (which is

     * sometimes itself mistreated as FTP). Yes, this will be

     * buggy. Why is it needed? For my personal coding experience and

     * because I don't know off-hand of any libraries that allow a

     * file upload to be interleaved over an existing connection with

     * its own protocol. Over TCP? We aren't doing real-time streaming

     * or anything :-p.

     *

     * Possible future expansions: ``transparent'' zlib compression.

     *

     * DATA: struct distren_request_file_post_start

     *

     * REQUIRED: DISTREN_SERVERTYPE_SUBMIT, DISTREN_SERVERTYPE_DISTRIBUTE

     */

    DISTREN_REQUEST_FILE_POST_START = 11,

    /**

     * file as directly to disk as possible to avoid bogging the

     * server down in swap. This packet may only be sent if

     * DISTREN_REQUEST_FILE_POST_START has previously been sent.

    DISTREN_REQUEST_FILE_POST = 12,

    /**

     * Marks a post-id's file as having completely uploaded. Provides

     * verification information so that the file's integrity may be

     * verified.

     *

     * DATA: struct distren_request_file_post_finish

     *

     * REQUIRED: DISTREN_SERVERTYPE_SUBMIT, DISTREN_SERVERTYPE_DISTRIBUTE

     */

    DISTREN_REQUEST_FILE_POST_FINISH = 13,

    DISTREN_REQUEST_FILE_FIND = 14,

    DISTREN_REQUEST_FILE = 15,

#define DISTREN_REQUEST_FILE_POST_NAME_LEN (64)

struct distren_request_file_post_start

{

  /**

   * Uniquely identify this file upload (per connection).

   */

  uint32_t post_id;

  /**

   * The user-friendly filename

   */

  char filename[DISTREN_REQUEST_FILE_POST_NAME_LEN];

};

#define DISTREN_REQUEST_FILE_POST_DATA_LEN (1024*128)

struct distren_request_file_post

{

  /**

   * Uniquely identify this file upload (per connection).

   */

  uint32_t post_id;

};

struct distren_request_file_post_finish

{

  /**

   * Uniquely identify this file upload (per connection).

   */

  uint32_t post_id;

  /**

   * Marks this upload as failed/cancelled.

   */

  uint32_t cancel;

  /**

   * An SHA1sum of the entire file. Only set if last_packet is

   * nonzero. Should be otherwise ignored.

   */

  unsigned char sha[EVP_MAX_MD_SIZE];

};

#include "common/request.h"

#include <arpa/inet.h>

#include <openssl/evp.h>

/*

 * file posting stuffs

 */

struct distren_request_file_post_context

{

  EVP_MD_CTX *digest_ctx;

  /* stored in network-byte order */

  uint32_t post_id;

};

/**

 * Helper for distren_request_file_post_start() and

 * distren_request_parse_file_post_start().

 */

static int _distren_request_file_post_context_new(distren_request_file_post_context_t *post_context,

						  uint32_t post_id)

{

  *post_context = malloc(sizeof(struct distren_request_file_post_context));

  if(!*post_context)

    return 1;

  (*post_context)->post_id = htonl(post_id);

  (*post_context)->digest_ctx = EVP_MD_CTX_create();

  EVP_DigestInit_ex((*post_context)->digest_ctx, EVP_sha1(), NULL);

  return 0;

}

/**

 * Frees a post_context and sets the pointer to NULL.

 */

static int _distren_request_file_post_context_free(distren_request_file_post_context_t *post_context)

{

  if(!*post_context)

    return 0;

  EVP_MD_CTX_destroy((*post_context)->digest_ctx);

  free(*post_context);

  *post_context = NULL;

  return 0;

}

int distren_request_file_post_context_free(distren_request_file_post_context_t post_context)

{

  /*

   * for the ``public'' function, we don't need to have post_context

   * set to NULL. Thus, we have this wrapper around

   * _distren_request_file_post_context_free().

   */

  return _distren_request_file_post_context_free(&post_context);

}

int distren_request_file_post_start(struct distren_request **req,

				    void **data,

				    distren_request_file_post_context_t *post_context,

				    uint32_t post_id,

				    const char *filename)

{

  struct distren_request_file_post_start *file_post_start;

  distren_request_new(req, sizeof(struct distren_request_file_post_start), DISTREN_REQUEST_FILE_POST_START);

  file_post_start = malloc(sizeof(struct distren_request_file_post_start));

  _distren_request_file_post_context_new(post_context, post_id);

  if(!*req

     || !file_post_start)

    {

      if(*req)

	distren_request_free(*req);

      free(file_post_start);

      _distren_request_file_post_context_free(post_context);

      return 1;

    }

  file_post_start->post_id = (*post_context)->post_id;

  strncpy(file_post_start->filename, filename, DISTREN_REQUEST_FILE_POST_NAME_LEN);

  *data = file_post_start;

  return 0;

}

int distren_request_parse_file_post_start(struct distren_request *req,

					  void *data,

					  distren_request_file_post_context_t *post_context,

					  uint32_t *post_id,

					  char **filename)

{

  int tmp;

  struct distren_request_file_post_start *file_post_start;

  /* keep our promises ;-) */

  *post_context = NULL;

  *filename = NULL;

  if(req->len < sizeof(struct distren_request_file_post_start))

    return 1;

  file_post_start = data;

  tmp = _distren_request_file_post_context_new(post_context, file_post_start->post_id);

  *filename = malloc(DISTREN_REQUEST_FILE_POST_NAME_LEN + 1);

  if(tmp

     || !*filename)

    {

      _distren_request_file_post_context_free(post_context);

      free(*filename);

    }

  memcpy(*filename, file_post_start->filename, DISTREN_REQUEST_FILE_POST_NAME_LEN);

  (*filename)[DISTREN_REQUEST_FILE_POST_NAME_LEN] = '\0';

  *post_id = ntohl(file_post_start->post_id);

  return 0;

}

int distren_request_file_post(struct distren_request **req,

			      struct distren_request_file_post *data_header,

			      distren_request_file_post_context_t post_context,

			      const void *data,

			      size_t len)

{

  distren_request_new(req, sizeof(struct distren_request_file_post) + len, DISTREN_REQUEST_FILE_POST);

  if(!*req)

    return 1;

  data_header->post_id = post_context->post_id;

  EVP_DigestUpdate(post_context->digest_ctx, data, len);

  return 0;

}

int distren_request_parse_file_post(struct distren_request *req,

				    void *data,

				    uint32_t *post_id,

				    distren_request_parse_file_post_find_context_func_t find_context_func,

				    void *find_post_context_data,

				    void **file_data,

				    size_t *file_data_len)

{

  struct distren_request_file_post *file_post;

  distren_request_file_post_context_t post_context;

  if(req->type != DISTREN_REQUEST_FILE_POST

     || req->len < sizeof(struct distren_request_file_post))

    return 1;

  file_post = data;

  *post_id = ntohl(file_post->post_id);

  post_context = (*find_context_func)(*post_id, find_post_context_data);

  if(!post_context

     || *post_id != post_context->post_id)

    return 1;

  *file_data = data + sizeof(struct distren_request_file_post);

  *file_data_len = req->len - sizeof(struct distren_request_file_post);

  EVP_DigestUpdate(post_context->digest_ctx, *file_data, *file_data_len);

  return 0;

}

int distren_request_file_post_finish(struct distren_request **req,

				     void **data,

				     distren_request_file_post_context_t post_context,

				     uint32_t cancel)

{

  struct distren_request_file_post_finish *file_post_finish;

  distren_request_new(req, sizeof(struct distren_request_file_post_finish), DISTREN_REQUEST_FILE_POST_FINISH);

  file_post_finish = malloc(sizeof(struct distren_request_file_post_finish));

  if(!*req

     || !file_post_finish)

    {

      if(*req)

	distren_request_free(*req);

      free(file_post_finish);

      _distren_request_file_post_context_free(&post_context);

      return 1;

    }

  file_post_finish->post_id = post_context->post_id;

  file_post_finish->cancel = htonl(cancel);

  EVP_DigestFinal(post_context->digest_ctx, file_post_finish->sha, NULL);

  _distren_request_file_post_context_free(&post_context);

  *data = file_post_finish;

  return 0;

}

int distren_request_parse_file_post_finish(struct distren_request *req,

				    void *data,

				    uint32_t *post_id,

				    distren_request_parse_file_post_find_context_func_t find_context_func,

				    void *find_post_context_data)

{

  struct distren_request_file_post_finish *file_post_finish;

  distren_request_file_post_context_t post_context;

  unsigned int checksum_len;

  unsigned char checksum[EVP_MAX_MD_SIZE];

  if(req->type != DISTREN_REQUEST_FILE_POST_FINISH

     || req->len < sizeof(struct distren_request_file_post_finish))

    return 1;

  file_post_finish = data;

  *post_id = ntohl(file_post_finish->post_id);

  if(ntohl(file_post_finish->cancel))

    return 2;

  post_context = find_context_func(*post_id, find_post_context_data);

  if(!post_context)

    return 1;

  /* finish up checksumming */

  EVP_DigestFinal_ex(post_context->digest_ctx, checksum, &checksum_len);

  if(checksum_len > EVP_MAX_MD_SIZE)

    checksum_len = EVP_MAX_MD_SIZE;

  if(memcmp(checksum, file_post_finish->sha, checksum_len))

    return 3;

  return 0;

}

#include <openssl/sha.h>

int distren_request_poing(struct distren_request **req, void **data, short is_ping, const void *poing_cookie, size_t poing_data_len);

struct distren_request_file_post_context;

typedef struct distren_request_file_post_context *distren_request_file_post_context_t;

/**

 * Frees a post_context for if a client is disconnected or an upload

 * otherwise interrupted.

 */

int distren_request_file_post_context_free(distren_request_file_post_context_t post_context);

/**

 * Prepares for a file upload and initializes a

 * distren_request_file_post_context_t.

 *

 * If you call this function and you never finish an upload, you

 * _must_ call distren_request_file_post_finish() on post_context at

 * least to free the post_context.

 *

 * @param req where to store the new distren_request pointer.

 * @param data where to store the new request's data pointer. 

 * @param post_context where the newly allocated distren_request_file_post_context_t's addrss should be stored.

 * @param post_id a number that uniquely identifies this upload for this connection.

 * @param filename a user-friendly filename for the file posting.

 */

int distren_request_file_post_start(struct distren_request **req,

				    void **data,

				    distren_request_file_post_context_t *post_context,

				    uint32_t post_id,

				    const char *filename);

/**

 * Parses a DISTREN_REQUEST_FILE_POST_START request.

 *

 * @param req the recieved request.

 * @param data the request's data.

 * @param post_context a post_context to be initialized for a continued download. Must be free()d with a call to distren_request_parse_file_post_finish() or distren_request_file_post_context_free(), however.

 * @param post_id will be set to the post_id contained in the packet.

 * @param filename will be made to point at the filename the client requested. *filename must be free()d.

 * @return 0 upon no problem, 1 if there's an error. If 1, then *filename should not be used (it will be set NULL, so no problem calling free() on it though).

 */

int distren_request_parse_file_post_start(struct distren_request *req,

					  void *data,

					  distren_request_file_post_context_t *post_context,

					  uint32_t *post_id,

					  char **filename);

/**

 * Initializes a DISTREN_REQUEST_FILE_POST packet's header.

 *

 * As file posting requires an acceptable chunk of memory, we don't

 * malloc() the data portion of the packet here to encourage reading

 * files in through static buffers.

 *

 * As a single file takes multiple DISTREN_REQUEST_FILE_POST packets

 * before it's completely uploaded, certain state information has to

 * persist between calls to distren_request_file_post(). This is what

 * the post_context argument is for. This pointer must be preserved

 * between calls to distren_request_file_post().

 *

 * @param req place to store the new req's pointer

 * @param data_header where the data's header should be stored

 * @param post_context what this points to must be preserved between calls to distren_request_file_post(), for keeping state.

FILE_POST_DATA_LEN.

 * @param data a hunk of the file being posted.

 * @param len the length of the data argument. This must be less than or equal to DISTREN_REQUEST_ */

int distren_request_file_post(struct distren_request **req,

			      struct distren_request_file_post *data_header,

			      distren_request_file_post_context_t post_context,

			      const void *data,

			      size_t len);

/**

 * A callback to find a distren_request_file_post_context_t for a

 * specified post_id.

 *

 * This exists because it is recognized that two servers may be

 * transfering more than one file at a time between eachother. At the

 * time that distren_request_parse_file_post() is called, the caller

 * doesn't know post_id and thus can't provide the proper

 * post_context. Yet distren_request_parse_file_post() needs the

 * post_context to get its job done ;-).

 *

 * @param post_id the post_id for which to find a distren_request_file_post_context_t

 * @param find_post_context_data the same find_post_context_data pointer passed to distren_request_parse_file_post() or distren_request_parse_file_post_finish().

 * @return NULL if the context could not be found or the context.

 */

typedef distren_request_file_post_context_t(*distren_request_parse_file_post_find_context_func_t)(uint32_t post_id, void *find_post_context_data);

/**

 * Parse a DISTREN_REQUEST_FILE_POST packet and update the

 * post_context.

 *

 * @param req the request header.

 * @param data the request data section.

 * @param post_id will be filled with the request's post_id.

 * @param find_context_func a function that can look up the necessary context_post given a post_id.

 * @param find_post_context_data the pointer to pass as context data to find_context_func

 * @param file_data will be set to the beginning of the file data portion of the packet. Do not free() this pointer.

 * @param file_data_len the number of bytes of file data that may be accessed at *file_data.

 * @return 0 on success and 1 on failure, in which case the client should be kicked and you should free the post_context with distren_request_file_post_context_free().

 */

int distren_request_parse_file_post(struct distren_request *req,

				    void *data,

				    uint32_t *post_id,

				    distren_request_parse_file_post_find_context_func_t find_context_func,

				    void *find_post_context_data,

				    void **file_data,

				    size_t *file_data_len);

/**

 * Construct a packet signalling that a file posting is finished.

 *

 * Frees the memory used to store the post_context. Must be called

 * once for every call to distren_request_file_post_start(),

 * regardless of whether or not you actually call

 * distren_request_send() on the result.

 *

 * @param req where to store the pointer of the newly allocated request

 * @param data where to store the pointer t othe newly allocated request data

 * @param post_context the post_context which will be used and then freed.

 * @param cancel 0 if the file was uploaded successfully, 1 if the upload is cancelled or failed. 

 */

int distren_request_file_post_finish(struct distren_request **req,

				     void **data,

				     distren_request_file_post_context_t post_context,

				     uint32_t cancel);

/**

 * Parses a DISTREN_REQUEST_FILE_POST_FINISH packet.

 *

 * You must still call distren_request_file_post_context_free() on the

 * post_context which you must find yourself. If you are a server, you

 * must keep a list of these contexts per-client so that when a client

 * exits, you may reclaim these contexts.

 *

 * @param req the request header

 * @param data the data section of the packet

 * @param post_id a place where the post_id will be set.

 * @param find_context_func a function pointer to let me find a post_context based on the post_id of the request.

 * @param find_post_context_data the pointer to pass to the find_context_func callback.

 * @return 0 for a good packet, 1 for a bad packet or if the post_context can't be found (post_id won't be valid), 2 if the request asks for the user to cancel the file upload (post_id will be valid), and 3 if the checksums don't match (post_id will be valid).

 */

int distren_request_parse_file_post_finish(struct distren_request *req,

				    void *data,

				    uint32_t *post_id,

				    distren_request_parse_file_post_find_context_func_t find_context_func,

				    void *find_post_context_data);

#include <arpa/inet.h>

    /**

     * where to store in-progress uploads or things that should

     * otherwise be on the same filesystem as the rest of the datadir

     * so that it may be rename()d.

     */

    char *tmpdir;

 * client request handlers

int distrend_handle_file_post_start(struct general_info *geninfo, struct distrend_client *client, struct distren_request *req, void *req_data);

int distrend_handle_file_post(struct general_info *geninfo, struct distrend_client *client, struct distren_request *req, void *req_data);

int distrend_handle_file_post_finish(struct general_info *geninfo, struct distrend_client *client, struct distren_request *req, void *req_data);

/* functions of some generic sort ...ish */

int distrend_handle_successful_upload(struct distrend_client *client, struct distrend_client_file_post *client_file_post);

  _distren_asprintf(&general_info.files.tmpdir, "%s/tmp",

		    general_info.config->datadir);

  distren_mkdir_recurse(general_info.files.tmpdir);

  distrend_listen_handler_add(general_info.config->listens, DISTREN_REQUEST_FILE_POST_START, &distrend_handle_file_post_start);

  distrend_listen_handler_add(general_info.config->listens, DISTREN_REQUEST_FILE_POST, &distrend_handle_file_post);

  distrend_listen_handler_add(general_info.config->listens, DISTREN_REQUEST_FILE_POST_FINISH, &distrend_handle_file_post_finish);

 * Traversal helper for distrend_client_find_post().

 */

int distrend_client_find_post_traverse(uint32_t *post_id, struct distrend_client_file_post *client_file_post)

{

  if(*post_id == client_file_post->post_id)

    return FALSE;

  return TRUE;

}

/**

 * Find the record for an in-progress client's file posting.

 */

struct distrend_client_file_post *distrend_client_find_post(struct distrend_client *client, uint32_t post_id)

{

  if(list_traverse(client->file_post_list, &post_id, (list_traverse_func_t)&distrend_client_find_post_traverse, LIST_ALTR | LIST_FORW | LIST_FRNT) != LIST_EXTENT)

    return list_curr(client->file_post_list);

  return NULL;

}

/**

 * Finds a post_context based on the post_id and client.

 *

 * Compatible the distren_request_parse_file_post_find_context_func_t.

 */

static distren_request_file_post_context_t distrend_client_find_file_post_context(uint32_t post_id, void *client)

{

  struct distrend_client_file_post *client_file_post;

  client_file_post = distrend_client_find_post(client, post_id);

  if(client_file_post)

    return client_file_post->post_context;

  return NULL;

}

/**

 * Clean up and free a client_file_post

 *

 * Whenever calling this functino, you almost _always_ have to call

 * list_remove_element(client->file_post_list, client_file_post);

 * first.

 */

void distrend_client_file_post_free(struct distrend_client_file_post *client_file_post)

{

  fclose(client_file_post->fd);

  free(client_file_post->filename);

  unlink(client_file_post->file_save_path);

  free(client_file_post->file_save_path);

  free(client_file_post);

}

int distrend_handle_file_post_start(struct general_info *geninfo, struct distrend_client *client, struct distren_request *req, void *req_data)

{

  struct distrend_client_file_post *client_file_post;

  distren_request_file_post_context_t post_context;

  uint32_t post_id;

  char *filename;

  char *str_tmp;

  int ret;

  /**

   * @todo access check!

   */

  fprintf(stderr, __FILE__ ":%d:distrend_handle_file_post_start(): You need to check if a client is actually allowed to upload files somehow!\n", __LINE__);

  /*