Changeset - 57766eb693c3
[Not reviewed]
default
0 3 0
Nathan Brink (binki) - 15 years ago 2010-06-29 23:13:02
ohnobinki@ohnopublishing.net
- Convert some of remoteio to use liblist.
- Fix bug where one function used counter in both an inner and outer loop (how embarassing ;-))
3 files changed with 44 insertions and 35 deletions:
0 comments (0 inline, 0 general)
etc/distrencommon.conf
Show inline comments
 
@@ -2,24 +2,32 @@
 
  Configuration file for distren.
 
  Currently, this file is being prepared as the goal for this project. For instance, the ability to support connecting and communicating with servers is suggested by the server sections.
 
*/
 

	
 
/*
 
  currently, server's are only supported rudimentarily. If a job has n frames and there are s servers, n / (s + 1) frames will be sent to each individual server and also rendered on the machine the job was submitted to. There will be no recursive distribution of jobs, but I want to make that possible in the future. AND, I want job IDs to have significance based on 1: the files, 2: the versions of software used to render just like git, mercurial, or bazaar's commit IDs have significance. This will allow global distribution of renderjobs without requiring central servers to coordinate the jobs - a network only need be distributed. And complex algorithms based on timeouts and completion of jobs should allow slow servers to reassign jobs to fast ones and, possibly, find shorter routes to return the resulting images to the original job submitter. Multiple server declarations are currently included, but an implementation for multiple servers is lacking.
 
*/
 

	
 
server protofusion
 
{
 
  hostname = "protofusion.org"
 
  username = "distrenc"
 
  method = "ssh"
 
  types = {"submit", "distribution"} /* submit means it accepts jobs, distribution means it can host files */
 
}
 

	
 
server ohnopublishing
 
{
 
  hostname = "ohnopublishing.net"
 
  username = "distrenc"
 
  method = "ssh"
 
  types = {"distribution"}
 
}
 

	
 

	
 
server localhost
 
{
 
  hostname = "localhost"
 
  username = "test"
 
  method = "tcp"
 
  types = {"submit", "distribution", "master"}
 
}
src/common/libremoteio.h
Show inline comments
 
/*
 
  Copyright 2010 Nathan Phillip Brink
 

	
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 
*/
 

	
 
#ifndef _DISTREN_LIBREMOTEIO_H
 
#define _DISTREN_LIBREMOTEIO_H
 

	
 
#include "remoteio.h"
 

	
 
#include <confuse.h>
 
#include <list.h>
 

	
 
/**
 
  private declarations for remoteio, to be included by options.c and remoteio.c
 
 */
 

	
 
enum remoteio_method
 
  {
 
    REMOTEIO_METHOD_SSH = 0,
 
#ifndef WINDOWS
 
    REMOTEIO_METHOD_UNIX = 1,
 
#endif
 
    REMOTEIO_METHOD_TCP = 2,
 
    /*    REMOETIO_METHOD_XMLRPC */ /*< maybe someday */
 
    REMOTEIO_METHOD_MAX = 3 /*< This is a number used to check the consitency of remoteio_server structs */
 
  };
 

	
 
struct remoteio_server
 
{
 
  struct remoteio_server *next;
 
  char *name; /*< The friendly named passed to remoteio_open() */
 
  char *hostname;
 
  char *username;
 
  enum remoteio_method method;
 
  unsigned int types; /*< See ``Server types'' in protocol.h */
 
};
 

	
 
struct remoteio_opts
 
{
 
  char *ssh_command;
 
  struct remoteio_server *servers;
 
  list_t servers;  /* type: (struct remoteio_server *) */
 
};
 

	
 
struct remoteio {
 
  enum remoteio_method method;
 
  struct remoteio_opts *opts;
 
  struct execio *execio;
 
#ifndef WINDOWS
 
  int sock;
 
#endif
 
};
 

	
 

	
 
/* lookup table */
 
typedef int remoteio_open_func_t(struct remoteio *rem, struct remoteio_server *server);
 
typedef int remoteio_read_func_t(struct remoteio *rem, void *buf, size_t len, size_t *bytesread);
 
typedef int remoteio_write_func_t(struct remoteio *rem, void *buf, size_t len, size_t *bytesread);
 
typedef int remoteio_close_func_t(struct remoteio *rem);
 

	
 
struct remoteio_method_funcmap
 
{
 
  enum remoteio_method method;
 

	
 
  remoteio_open_func_t *open_func;
 
  remoteio_read_func_t *read_func;
src/common/remoteio.c
Show inline comments
 
/*
 
  Copyright 2010 Nathan Phillip Brink
 

	
 
  This file is a part of DistRen.
 

	
 
  DistRen is free software: you can redistribute it and/or modify
 
  it under the terms of the GNU Affero General Public License as published by
 
  the Free Software Foundation, either version 3 of the License, or
 
  (at your option) any later version.
 

	
 
  DistRen is distributed in the hope that it will be useful,
 
  but WITHOUT ANY WARRANTY; without even the implied warranty of
 
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
  GNU Affero General Public License for more details.
 

	
 
  You should have received a copy of the GNU Affero General Public License
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
*/
 

	
 
#include "libremoteio.h"
 
#include "execio.h"
 
#include "asprintf.h"
 

	
 
#include <list.h>
 

	
 
#include <errno.h>
 
#ifndef _WIN32
 
#include <netdb.h>
 
#endif
 
#include <stdlib.h>
 
#include <stdio.h>
 
#include <string.h>
 
#include <sys/types.h>
 
#ifdef _WIN32
 
#include <winsock2.h>
 
#include <ws2tcpip.h>
 
#else
 
#include <sys/socket.h>
 
#endif
 
#include <unistd.h>
 

	
 
#ifndef _WIN32
 
#include <sys/un.h>
 
#endif
 

	
 
/* local */
 

	
 
#define REMOTEIO_DEFAULT_PORT "4050"
 

	
 
@@ -62,103 +64,92 @@ int _remoteio_tcp_open(struct remoteio *
 
int _remoteio_tcp_close(struct remoteio *rem);
 

	
 
/**
 
  lookup table for different methods of remoteio:
 
  the enum remoteio_method is the index of the entry to use for that method. 
 
  Regardless, a NULL terminator is required because the configuration function
 
  searches through this table for the method specified in the config file.
 
*/
 
struct remoteio_method_funcmap funcmap[] = 
 
  {
 
    /* [REMOTEIO_METHOD_SSH] */
 
    {REMOTEIO_METHOD_SSH, &_remoteio_ssh_open, &_remoteio_ssh_read, &_remoteio_ssh_write, &_remoteio_ssh_close, "ssh"},
 
#ifndef _WIN32
 
    {REMOTEIO_METHOD_UNIX, &_remoteio_sock_open, &_remoteio_sock_read, &_remoteio_sock_write, &_remoteio_sock_close, "unix"},
 
#endif
 
    {REMOTEIO_METHOD_TCP, &_remoteio_tcp_open, &_remoteio_sock_read, &_remoteio_sock_write, &_remoteio_tcp_close, "tcp"},
 
    {REMOTEIO_METHOD_MAX, NULL, NULL, NULL, NULL, NULL}
 
  };
 

	
 
struct remoteio_server *remoteio_getserver(const struct remoteio_opts *opts, const char *servername);
 

	
 
int remoteio_config(cfg_t *cfg, struct remoteio_opts *opts)
 
{
 
  size_t numservers;
 
  size_t counter;
 
  size_t counter, counter2;
 
  static int haslisted_methods = 0;
 
  
 
  struct remoteio_server *aserver;
 
  struct remoteio_server aserver;
 

	
 
  opts->servers = malloc(sizeof(struct remoteio_server));
 
  opts->servers = list_init();
 
  if(!opts->servers)
 
    {
 
      fprintf(stderr, "@todo cleanup!\n");
 
      abort();
 
    }
 
  
 
  aserver = opts->servers;
 
  
 
  numservers = cfg_size(cfg, "server"); 
 
  numservers = cfg_size(cfg, "server");
 
  for(counter = 0; counter < numservers; counter ++)
 
    {
 
      cfg_t *cfg_aserver;
 
      char *method;
 
      
 
      cfg_aserver = cfg_getnsec(cfg, "server", counter);
 
      
 
      if(!aserver) /*< if the malloc in the previous loop failed */
 
	abort();
 
      
 
      aserver->name = strdup(cfg_title(cfg_aserver));
 
      aserver->hostname = strdup(cfg_getstr(cfg_aserver, "hostname"));
 
      aserver->username = strdup(cfg_getstr(cfg_aserver, "username"));
 
      aserver.name = strdup(cfg_title(cfg_aserver));
 
      aserver.hostname = strdup(cfg_getstr(cfg_aserver, "hostname"));
 
      aserver.username = strdup(cfg_getstr(cfg_aserver, "username"));
 

	
 
      aserver->method = REMOTEIO_METHOD_MAX;
 
      aserver.method = REMOTEIO_METHOD_MAX;
 
      method = cfg_getstr(cfg_aserver, "method");
 
      for(counter = 0; funcmap[counter].name; counter ++)
 
	if(strcmp(method, funcmap[counter].name) == 0)
 
	  aserver->method = REMOTEIO_METHOD_SSH;
 
      if(aserver->method == REMOTEIO_METHOD_MAX)
 
      for(counter2 = 0; funcmap[counter2].name; counter2 ++)
 
	if(strcmp(method, funcmap[counter2].name) == 0)
 
	  aserver.method = REMOTEIO_METHOD_SSH;
 
      if(aserver.method == REMOTEIO_METHOD_MAX)
 
	{
 
	  fprintf(stderr, "No such method as %s\n", method);
 
	  if(!haslisted_methods)
 
	    {
 
	      fprintf(stderr, "Available methods:\n");
 
	      for(counter = 0; funcmap[counter].name; counter ++)
 
		fprintf(stderr, "\t%s\n", funcmap[counter].name);
 
	      for(counter2 = 0; funcmap[counter2].name; counter2 ++)
 
		fprintf(stderr, "\t%s\n", funcmap[counter2].name);
 
	      
 
	      haslisted_methods ++;
 
	    }
 
	  abort();
 
	}
 
      
 
      if(counter < numservers - 1)
 
	{
 
	  aserver->next = malloc(sizeof(struct remoteio_server));
 
	  aserver = aserver->next;
 
	}
 
      list_insert_after(opts->servers, &aserver, sizeof(struct remoteio_server));
 
    }
 
  aserver->next = NULL;
 
  
 
  return 0;
 
}
 

	
 

	
 

	
 
int remoteio_open(struct remoteio **remoteio, struct remoteio_opts *opts, const char *servername)
 
{
 
  struct remoteio_server *theserver;
 
  struct remoteio *rem;
 

	
 
  int tmp;
 

	
 
  if(!opts)
 
    {
 
      fprintf(stderr, "%s:%d: no null opts!\n\tThis is a bug, please report it (after making sure it isn't already reported)\n", __FILE__, __LINE__);
 
      return 1;
 
    }
 

	
 
  theserver = remoteio_getserver(opts, servername);
 
  if(!theserver)
 
    {
 
      fprintf(stderr, "%s:%d: Could not find server named ``%s''\n", __FILE__, __LINE__, servername);
 
      return 1;
 
@@ -192,58 +183,67 @@ int remoteio_open(struct remoteio **remo
 
  
 
  return 0;
 
}
 

	
 
int remoteio_read(struct remoteio *rem, void *buf, size_t len, size_t *bytesread)
 
{
 
  return funcmap[rem->method].read_func(rem, buf, len, bytesread);
 
}
 

	
 
int remoteio_write(struct remoteio *rem, void *buf, size_t len, size_t *byteswritten)
 
{
 
  return funcmap[rem->method].write_func(rem, buf, len, byteswritten);
 
}
 

	
 
int remoteio_close(struct remoteio *rem)
 
{
 
  int rtn;
 
  
 
  rtn = funcmap[rem->method].close_func(rem);
 
  free(rem);
 
  
 
  return rtn;
 
}
 

	
 
int _remoteio_getserver_traverse(char *servername, struct remoteio_server *aserver)
 
{
 
  if(!strcmp(aserver->name, servername))
 
    return FALSE; /* stop traversal */
 

	
 
  return TRUE;
 
}
 

	
 
struct remoteio_server *remoteio_getserver(const struct remoteio_opts *opts, const char *servername)
 
{
 
  struct remoteio_server *aserver;
 
  
 
  for(aserver = opts->servers;
 
      aserver;
 
      aserver = aserver->next)
 
    if(!strcmp(servername, aserver->name))
 
      return aserver;
 
  int traversal_result;
 
  char *dispensible_servername;
 

	
 
  dispensible_servername = strdup(servername); /* for the sake of constness... */
 
  traversal_result = list_traverse(opts->servers, dispensible_servername, (list_traverse_func_t)&_remoteio_getserver_traverse, LIST_FRNT | LIST_ALTR);
 
  free(dispensible_servername);
 

	
 
  if(traversal_result == LIST_OK)
 
    return (struct remoteio_server *)list_curr(opts->servers);
 

	
 
  return (struct remoteio_server *)NULL;
 
}
 

	
 

	
 

	
 

	
 
/**
 
   different remoteio methods' implementations:
 
 */
 

	
 
/*
 
  SSH, via execio
 
*/
 

	
 
int _remoteio_ssh_open(struct remoteio *rem, struct remoteio_server *server)
 
{
 
  char *userhost;
 
  char *sshargs[] = {rem->opts->ssh_command, NULL /* userhost */, "distrend", "-d", (char *)NULL};
 

	
 
  int rtn;
 

	
 
  if(server->username)
 
    _distren_asprintf(&userhost, "%s@%s", server->username, server->hostname);
0 comments (0 inline, 0 general)