Changeset - 754555c90530
[Not reviewed]
default
0 2 0
ethanzonca - 16 years ago 2009-09-03 22:39:53

Some structural updates and minor changes in a readthrough of the server code.
2 files changed with 24 insertions and 57 deletions:
0 comments (0 inline, 0 general)
TODO
Show inline comments
 
@@ -10,13 +10,13 @@ Build
 

	
 
Master
 
*-Add calls to remotio
 
 -Write more meat into the pseudo-code main() to pull everything together
 
 -Write a stub for getting info from the tarball/validifying the tarball. Read distren-job.xml, a file in the tarball, to find out 1. which rendering system to use (that system, e.g. blender/povray, can read more specifics, such as name of file to pass to blender and frames. Options common between different systems will be handled in common as best as possible)
 
 -Write a stub for publishing file and constructing job description so that the job can be shared
 
*-Other XML-writing/reading
 
*-Other XML-writing/reading such as current stats / jobs / data / etc, xml writing for job creation, ...
 
*-charsets: the program should operate in UTF-8. This is partially required by libxml2's (and XML's) usage of UTF-8. We want the program's internal charset to be UTF-8 and figure out if we need w_char/whatever to fulfill UTF-8. I don't believe we do... mabye we just need a declaration that al data stored must be in UTF-8 format.
 
 -rename some struct members and variables to be more program-agnostic and more sensible (e.g., no blender-specific stuff)
 

	
 
Slave
 
 -Fix current bugs
 
*-Add calls to remoteio, once it works or even before it works
 
@@ -25,12 +25,13 @@ Slave
 
=-Make code more flexible for different types of jobs and operating systems (such as paths, libs, etc)
 
 -Add code to allow pausing/resuming of the slave (including a running blender process)
 
 -Add a threads variable for software that doesn't support threads (blender takes care of this by itself)
 
*-Fix line 194 (call to myjob struct that fails miserably)
 
B-Add tmpdir variable (just like the datadir variable) that is compile-time or whatever
 
*-Add code to delete old jobfiles (maybe based on timestamp, etc... make sure they're uploaded first)
 
 -Upload while rendering? Would be nice.
 

	
 
Options
 
 -Rewrite some stuff, try to make it simpler
 
x-Move server and client confuse code into their individual files, rather than in the common file. -- note: there wasn't any client code in options.c. There was the ``server'' section. That section is passed multiple times and provides information to remoteio on how to connect to servers
 
 -Review all confuse interfacing
 
 -Push patch for relative includes in confuse to the confuse developers
src/server/distrend.c
Show inline comments
 
@@ -17,42 +17,12 @@
 
  along with DistRen.  If not, see <http://www.gnu.org/licenses/>.
 

	
 
*/
 

	
 
/* This file contains the code which both processes (renders) jobs as a slave, and the code which distributes frames to slaves after receiving them from the client portion of the codebase. */
 

	
 
 /* Just some notes -- Ethan Zonca
 
 * ++ Make data availible for other apps to parse
 
 * Server<==>Client Communication
 
 * Upload while rendering
 
 */
 

	
 

	
 
/* XML notes --
 
 *
 
 * First off, is XML in our build environment?
 
 * Next, what files are we going to have?:
 
 *  - XML file sent with (.blend/.pov/.lux(xml)/whatever) by distren telling the server what to do
 
 *  	+ Includes submitter name, filename, emailaddress, etc
 
 *  - XML file on server storing info on current and finished jobs
 
 *  - XML file sent by distren when submitting a render to the server
 

	
 
 *
 
 *  Overall, we could use XML to make our client/server communication load
 
 *  a lot lighter, especially if clients are sent customized xml files telling
 
 *  them what frame(s) to render... although that would be inefficient if rendering
 
 *  single frames. Maybe not tell it what frames to render in the xml file, the server
 
 *  can spit out that info.
 
 *
 
 *  Please edit my ramblings if you please,
 
 *  --ethanzonca
 
 *
 
 */
 

	
 

	
 

	
 
#include "execio.h"
 
#include "options.h"
 
#include "distrenjob.h"
 
#include "protocol.h"
 
#include "slavefuncs.h"
 

	
 
@@ -61,18 +31,15 @@
 
#include <stdlib.h>
 
#include <malloc.h>
 
#include <unistd.h> /* getopt */
 
#include <time.h>
 

	
 

	
 

	
 

	
 

	
 
/* ******************* Structs ************************ */
 

	
 
// Stores config info? editmycomment
 
// Gets config info from confs
 
struct distrend_config
 
{
 
  cfg_t *mycfg;
 
  struct options_common *options;
 
  struct distrend_listen **listens; /*< Null terminated array of structs */
 
  char *datadir;
 
@@ -84,15 +51,12 @@ struct distrend_config
 
 frame[frame] Status Assignments:
 
  "NULL" - don't render me
 
  "0" - canceled
 
  "1" - unassigned
 
  "2" - assigned to slave
 
  "3" - completed by slave and uploaded
 

	
 
 Have a script crawl through each job in the arrays, priority-biased, and assign a frame to each slave.
 
 Then we will need some sort of watchdog to monitor slaves on the main server to check for stales. Maybe not worry about this for now.
 
*/
 

	
 
struct general_info {
 
  short int jobs_in_queue; //
 
  unsigned short int free_clients;
 
  unsigned short int rendering_clients;//
 
@@ -115,14 +79,12 @@ jobnum_t jobnum = 0; // The next job num
 
int hcfjob; // Highest consecutively finished job
 
int highest_jobnum; // The job number of the most recently created job, this is used when creating new jobs
 

	
 

	
 
/* ********************** Functions ************************* */
 

	
 
/* Functions to stubify */
 

	
 
/**
 
   Performs command stored in a client's request.
 
*/
 
int distrend_do()
 
{
 
  return 0;
 
@@ -160,46 +122,52 @@ void distrend_unlisten()
 
*/
 
void remotio_send_to_client()
 
{
 
	// I am futile!
 
}
 
/** Fill variables after crash / shutdown from XML dumps */
 
void start_data(){
 
int start_data(){
 
  if(1 == 0){
 
    // retrieve total_finished_jobs and total_finished_frames from xml file
 
    fprintf(stderr,"Parsing XML files and restoring previous state...\n");
 
    return 1;
 
  }
 
  else{
 
    general_info.total_finished_jobs = 0;
 
    general_info.total_frames_rendered = 0;
 
    fprintf(stderr,"Can't find XML dump, starting up fresh.\n");
 
    return 2;
 
  }
 
}
 

	
 
/** Finish-Setter: Sets a frame to the "completed" status.*/
 
void finish_frame(struct distrenjob *distrenjob, int frame){
 
  distrenjob->frameset[frame].status = 2;
 
  distrenjob->frameset[frame].time_to_render = (clock() - distrenjob[jobnum].frameset[frame].start_time); // Consider changing time-to-render to time-for-frame or something?
 

	
 
  general_info.total_frames_rendered++; // Increase total frames var for stats
 
}
 

	
 

	
 
/** Queuer: Adds files to the queue */
 
void queue(struct distrenjob *distrenjob, int type, char *name, char *submitter, char *email, int priority, int mode, int spp, struct frameset *frameset) {
 
int queue(struct distrenjob *distrenjob, int type, char *name, char *submitter, char *email, int priority, int mode, int spp, struct frameset *frameset) {
 
  // Type: 1 = blender, add more types later
 
  // jobnum is the next available job number
 
  if(type == 1){
 
    distrenjob->name = name;
 
    distrenjob->submitter = submitter;
 
    distrenjob->email = email;
 
    distrenjob->priority = priority;
 
    distrenjob->frameset = frameset;
 
  }
 
  else{
 
    // Throw error.
 
    fprintf(stderr,"You tried to queue a job type that isn't supported by this version of DistRen.\nI have no idea how this would happen, but it just did.\n");
 
    return 0; // fail
 
  }
 
jobnum++; // Advance the jobnumber for the next queued job
 
return 0; // OK
 
}
 

	
 

	
 
/**
 
  Status Report Generator:
 
  -figures out how much of the job is done, where jobnum corresponds to the job number
 
@@ -323,13 +291,13 @@ void frame_num_struct_builder(struct dis
 

	
 
  @return 0 success, other: error
 
*/
 
int frame_finder(struct distrenjob *head, struct distrenjob **job, struct frameset **frame)
 
{
 
  int your_frame;  // your_frame is an integer value that will be given to the client as the frame number to render
 
  // UNUSED: int your_job;	   // @TODO: Fixme: do we need this var? you_job is an integer value that must ALSO be given to the client
 
  int your_job;	   // @TODO: Fixme: do we need this var? you_job is an integer value that must ALSO be given to the client
 

	
 
  unsigned short int found;
 
  unsigned short int priority;
 

	
 
  struct distrenjob *distrenjob_ptr;
 

	
 
@@ -370,34 +338,33 @@ int frame_finder(struct distrenjob *head
 
	  /* If that job had no open frames for some reason, run the status report generator so that */
 
          status_report_generator(&head);
 

	
 
          /* should the job be removed now? */
 
          fprintf(stderr, "Job %d is finished, this is probably the place to call the job-removal function\n", distrenjob_ptr->jobnum);
 
        }
 
    } /* while(!found) */
 
    }
 

	
 
  fprintf(stderr, "Missing apostrophe !!!!!!!!!!!!!!\n"); abort();
 
  /* sets the value of the frame to 1, which means its taken !!!!!! MISSSING APOSTROPHE!!!!!!! */
 
  distrenjob_ptr->frameset[your_frame].status++;
 

	
 
  distrenjob_ptr->frameset[your_frame].start_time = clock();
 

	
 
  *job = distrenjob_ptr;
 
  *frame =  &distrenjob_ptr->frameset[your_frame];
 

	
 
  return 0;
 
}
 

	
 
/** Checks for dying slaves */
 
/** Checks for dead, laggy, or stale slaves */
 
void blend_frame_watchdog(struct distrenjob *distrenjob_head)
 
{
 
  unsigned short int watchdog_forgiveness; /*< seconds to wait on a frame before re-assigning it */
 
  struct distrenjob *distrenjob_ptr;
 
  unsigned int counter;
 

	
 
  watchdog_forgiveness = 3; /*< hours of forgiveness before frame is re-assigned */
 
  watchdog_forgiveness = 3; /*< hours of forgiveness before frame is re-assigned @TODO: Make this more user-configurable (maybe per-job), 3 hours is a LONG time */
 
  distrenjob_ptr = distrenjob_head;
 

	
 
  for(distrenjob_ptr = distrenjob_head; distrenjob_ptr; distrenjob_ptr = distrenjob_ptr->next)
 
    /* iterate through jobs */
 

	
 
    for(counter = 0; counter < distrenjob_ptr->total_frames; counter ++)
 
@@ -432,13 +399,13 @@ struct distrenjob *distrenjob_get(struct
 

	
 
  return distrenjob_ptr;
 
}
 

	
 

	
 
/**
 
   Removes a distrenjob from the distrenjob linkelist.
 
   Removes a distrenjob from the distrenjob linked list.
 

	
 
   @arg head a double pointer. the head pointer will have to be changed if distrenjob == *head. Thus, make sure that the pointer points to the pointer to the head that all functions use. (I'm going to come back to this and misunderstand myself ;-))
 
 */
 
void distrenjob_remove(struct distrenjob **head, struct distrenjob *bj)
 
{
 
  struct distrenjob *previous_distrenjob;
 
@@ -464,16 +431,13 @@ void distrenjob_remove(struct distrenjob
 
     @lordofwar: the magic deallocation of memory ;-)
 
   */
 
  free(bj);
 
}
 

	
 

	
 
/* ************************** Main ************************* */
 

	
 

	
 
// Begin non-working framework?
 
/* Grabs config info from confs */
 
int distrend_do_config(int argc, char *argv[], struct distrend_config **config)
 
{
 
  cfg_opt_t myopts_listen[] =
 
    {
 
      CFG_SIMPLE_STR("type", NULL),
 
      CFG_SIMPLE_STR("path", NULL),
 
@@ -497,13 +461,13 @@ int distrend_do_config(int argc, char *a
 
  /*
 
   * test xml2distrenjob()
 
   */
 
  tmp = xml2distrenjob(&distrenjob, "distrenjob.xml.example");
 
  if(tmp)
 
    fprintf(stderr, "xml2distrenjob() returned %d. Try to cd to distren/doc if you want to test out the xml2distrenjob() function. (This will only fix this error if the error is due to an inability of the xml library to access distrenjob.xml.example)\n", tmp);
 
  else  
 
  else
 
    fprintf(stderr, "using email ``%s'' for user ``%s'' -- reading in XML files and pulling data from them using libxml2+XPath works!!!\n", distrenjob->email, distrenjob->submitter);
 

	
 
  fprintf(stderr, "%s:%d running config\n", __FILE__, __LINE__);
 

	
 
  *config = malloc(sizeof(struct distrend_config));
 
  myopts[1].simple_value = &(*config)->datadir;
 
@@ -519,26 +483,28 @@ int distrend_config_free(struct distrend
 
{
 
  options_free(config->options);
 
  free(config);
 

	
 
  return 0;
 
}
 
// End non-working framework?
 

	
 

	
 
/* ************************** Main ************************* */
 

	
 
int main(int argc, char *argv[])
 
{
 

	
 
  /* @TODO: Put some arg-grabbing code here */
 

	
 
  struct distrenjob *head;
 

	
 
  int cont;
 
  struct distrend_listenset *listenset;
 
  struct distrend_config *config;
 

	
 
  start_data(); // Starts fresh or loads data from xml dump
 
  start_data(); // Starts fresh or loads data from xml dump. Should we grab the return?
 

	
 
  enum clientstatus
 
  {
 
    CLIENTSTATUS_UNINITIALIZED = 0,
 
    CLIENTSTATUS_BUSY = 1,
 
    CLIENTSTATUS_IDLE = 2
 
@@ -547,13 +513,13 @@ int main(int argc, char *argv[])
 
  head = NULL;
 
  cont = 1;
 

	
 
  distrend_do_config(argc, argv, &config);
 

	
 
  distrend_listen(&listenset, config);
 
  /* This is called the ``main loop'' */
 
  /* This is called the "main loop" */
 
  while(cont)
 
    {
 
      struct distren_action *action;
 
      int clientsays; /*< temporary example variable, will be replaced when we can handle messages */
 

	
 
      distrend_accept(&action);
0 comments (0 inline, 0 general)